1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2013-2017 ARM Limited, All Rights Reserved.
4 * Author: Marc Zyngier <marc.zyngier@arm.com>
5 */
6
7 #include <linux/acpi.h>
8 #include <linux/acpi_iort.h>
9 #include <linux/bitfield.h>
10 #include <linux/bitmap.h>
11 #include <linux/cpu.h>
12 #include <linux/crash_dump.h>
13 #include <linux/delay.h>
14 #include <linux/efi.h>
15 #include <linux/interrupt.h>
16 #include <linux/iommu.h>
17 #include <linux/iopoll.h>
18 #include <linux/irqdomain.h>
19 #include <linux/list.h>
20 #include <linux/log2.h>
21 #include <linux/memblock.h>
22 #include <linux/mm.h>
23 #include <linux/msi.h>
24 #include <linux/of.h>
25 #include <linux/of_address.h>
26 #include <linux/of_irq.h>
27 #include <linux/of_pci.h>
28 #include <linux/of_platform.h>
29 #include <linux/percpu.h>
30 #include <linux/slab.h>
31 #include <linux/syscore_ops.h>
32
33 #include <linux/irqchip.h>
34 #include <linux/irqchip/arm-gic-v3.h>
35 #include <linux/irqchip/arm-gic-v4.h>
36
37 #include <asm/cputype.h>
38 #include <asm/exception.h>
39
40 #include "irq-gic-common.h"
41
42 #define ITS_FLAGS_CMDQ_NEEDS_FLUSHING (1ULL << 0)
43 #define ITS_FLAGS_WORKAROUND_CAVIUM_22375 (1ULL << 1)
44 #define ITS_FLAGS_WORKAROUND_CAVIUM_23144 (1ULL << 2)
45 #define ITS_FLAGS_FORCE_NON_SHAREABLE (1ULL << 3)
46
47 #define RD_LOCAL_LPI_ENABLED BIT(0)
48 #define RD_LOCAL_PENDTABLE_PREALLOCATED BIT(1)
49 #define RD_LOCAL_MEMRESERVE_DONE BIT(2)
50
51 static u32 lpi_id_bits;
52
53 /*
54 * We allocate memory for PROPBASE to cover 2 ^ lpi_id_bits LPIs to
55 * deal with (one configuration byte per interrupt). PENDBASE has to
56 * be 64kB aligned (one bit per LPI, plus 8192 bits for SPI/PPI/SGI).
57 */
58 #define LPI_NRBITS lpi_id_bits
59 #define LPI_PROPBASE_SZ ALIGN(BIT(LPI_NRBITS), SZ_64K)
60 #define LPI_PENDBASE_SZ ALIGN(BIT(LPI_NRBITS) / 8, SZ_64K)
61
62 #define LPI_PROP_DEFAULT_PRIO GICD_INT_DEF_PRI
63
64 /*
65 * Collection structure - just an ID, and a redistributor address to
66 * ping. We use one per CPU as a bag of interrupts assigned to this
67 * CPU.
68 */
69 struct its_collection {
70 u64 target_address;
71 u16 col_id;
72 };
73
74 /*
75 * The ITS_BASER structure - contains memory information, cached
76 * value of BASER register configuration and ITS page size.
77 */
78 struct its_baser {
79 void *base;
80 u64 val;
81 u32 order;
82 u32 psz;
83 };
84
85 struct its_device;
86
87 /*
88 * The ITS structure - contains most of the infrastructure, with the
89 * top-level MSI domain, the command queue, the collections, and the
90 * list of devices writing to it.
91 *
92 * dev_alloc_lock has to be taken for device allocations, while the
93 * spinlock must be taken to parse data structures such as the device
94 * list.
95 */
96 struct its_node {
97 raw_spinlock_t lock;
98 struct mutex dev_alloc_lock;
99 struct list_head entry;
100 void __iomem *base;
101 void __iomem *sgir_base;
102 phys_addr_t phys_base;
103 struct its_cmd_block *cmd_base;
104 struct its_cmd_block *cmd_write;
105 struct its_baser tables[GITS_BASER_NR_REGS];
106 struct its_collection *collections;
107 struct fwnode_handle *fwnode_handle;
108 u64 (*get_msi_base)(struct its_device *its_dev);
109 u64 typer;
110 u64 cbaser_save;
111 u32 ctlr_save;
112 u32 mpidr;
113 struct list_head its_device_list;
114 u64 flags;
115 unsigned long list_nr;
116 int numa_node;
117 unsigned int msi_domain_flags;
118 u32 pre_its_base; /* for Socionext Synquacer */
119 int vlpi_redist_offset;
120 };
121
122 #define is_v4(its) (!!((its)->typer & GITS_TYPER_VLPIS))
123 #define is_v4_1(its) (!!((its)->typer & GITS_TYPER_VMAPP))
124 #define device_ids(its) (FIELD_GET(GITS_TYPER_DEVBITS, (its)->typer) + 1)
125
126 #define ITS_ITT_ALIGN SZ_256
127
128 /* The maximum number of VPEID bits supported by VLPI commands */
129 #define ITS_MAX_VPEID_BITS \
130 ({ \
131 int nvpeid = 16; \
132 if (gic_rdists->has_rvpeid && \
133 gic_rdists->gicd_typer2 & GICD_TYPER2_VIL) \
134 nvpeid = 1 + (gic_rdists->gicd_typer2 & \
135 GICD_TYPER2_VID); \
136 \
137 nvpeid; \
138 })
139 #define ITS_MAX_VPEID (1 << (ITS_MAX_VPEID_BITS))
140
141 /* Convert page order to size in bytes */
142 #define PAGE_ORDER_TO_SIZE(o) (PAGE_SIZE << (o))
143
144 struct event_lpi_map {
145 unsigned long *lpi_map;
146 u16 *col_map;
147 irq_hw_number_t lpi_base;
148 int nr_lpis;
149 raw_spinlock_t vlpi_lock;
150 struct its_vm *vm;
151 struct its_vlpi_map *vlpi_maps;
152 int nr_vlpis;
153 };
154
155 /*
156 * The ITS view of a device - belongs to an ITS, owns an interrupt
157 * translation table, and a list of interrupts. If it some of its
158 * LPIs are injected into a guest (GICv4), the event_map.vm field
159 * indicates which one.
160 */
161 struct its_device {
162 struct list_head entry;
163 struct its_node *its;
164 struct event_lpi_map event_map;
165 void *itt;
166 u32 nr_ites;
167 u32 device_id;
168 bool shared;
169 };
170
171 static struct {
172 raw_spinlock_t lock;
173 struct its_device *dev;
174 struct its_vpe **vpes;
175 int next_victim;
176 } vpe_proxy;
177
178 struct cpu_lpi_count {
179 atomic_t managed;
180 atomic_t unmanaged;
181 };
182
183 static DEFINE_PER_CPU(struct cpu_lpi_count, cpu_lpi_count);
184
185 static LIST_HEAD(its_nodes);
186 static DEFINE_RAW_SPINLOCK(its_lock);
187 static struct rdists *gic_rdists;
188 static struct irq_domain *its_parent;
189
190 static unsigned long its_list_map;
191 static u16 vmovp_seq_num;
192 static DEFINE_RAW_SPINLOCK(vmovp_lock);
193
194 static DEFINE_IDA(its_vpeid_ida);
195
196 #define gic_data_rdist() (raw_cpu_ptr(gic_rdists->rdist))
197 #define gic_data_rdist_cpu(cpu) (per_cpu_ptr(gic_rdists->rdist, cpu))
198 #define gic_data_rdist_rd_base() (gic_data_rdist()->rd_base)
199 #define gic_data_rdist_vlpi_base() (gic_data_rdist_rd_base() + SZ_128K)
200
201 /*
202 * Skip ITSs that have no vLPIs mapped, unless we're on GICv4.1, as we
203 * always have vSGIs mapped.
204 */
require_its_list_vmovp(struct its_vm * vm,struct its_node * its)205 static bool require_its_list_vmovp(struct its_vm *vm, struct its_node *its)
206 {
207 return (gic_rdists->has_rvpeid || vm->vlpi_count[its->list_nr]);
208 }
209
rdists_support_shareable(void)210 static bool rdists_support_shareable(void)
211 {
212 return !(gic_rdists->flags & RDIST_FLAGS_FORCE_NON_SHAREABLE);
213 }
214
get_its_list(struct its_vm * vm)215 static u16 get_its_list(struct its_vm *vm)
216 {
217 struct its_node *its;
218 unsigned long its_list = 0;
219
220 list_for_each_entry(its, &its_nodes, entry) {
221 if (!is_v4(its))
222 continue;
223
224 if (require_its_list_vmovp(vm, its))
225 __set_bit(its->list_nr, &its_list);
226 }
227
228 return (u16)its_list;
229 }
230
its_get_event_id(struct irq_data * d)231 static inline u32 its_get_event_id(struct irq_data *d)
232 {
233 struct its_device *its_dev = irq_data_get_irq_chip_data(d);
234 return d->hwirq - its_dev->event_map.lpi_base;
235 }
236
dev_event_to_col(struct its_device * its_dev,u32 event)237 static struct its_collection *dev_event_to_col(struct its_device *its_dev,
238 u32 event)
239 {
240 struct its_node *its = its_dev->its;
241
242 return its->collections + its_dev->event_map.col_map[event];
243 }
244
dev_event_to_vlpi_map(struct its_device * its_dev,u32 event)245 static struct its_vlpi_map *dev_event_to_vlpi_map(struct its_device *its_dev,
246 u32 event)
247 {
248 if (WARN_ON_ONCE(event >= its_dev->event_map.nr_lpis))
249 return NULL;
250
251 return &its_dev->event_map.vlpi_maps[event];
252 }
253
get_vlpi_map(struct irq_data * d)254 static struct its_vlpi_map *get_vlpi_map(struct irq_data *d)
255 {
256 if (irqd_is_forwarded_to_vcpu(d)) {
257 struct its_device *its_dev = irq_data_get_irq_chip_data(d);
258 u32 event = its_get_event_id(d);
259
260 return dev_event_to_vlpi_map(its_dev, event);
261 }
262
263 return NULL;
264 }
265
vpe_to_cpuid_lock(struct its_vpe * vpe,unsigned long * flags)266 static int vpe_to_cpuid_lock(struct its_vpe *vpe, unsigned long *flags)
267 {
268 raw_spin_lock_irqsave(&vpe->vpe_lock, *flags);
269 return vpe->col_idx;
270 }
271
vpe_to_cpuid_unlock(struct its_vpe * vpe,unsigned long flags)272 static void vpe_to_cpuid_unlock(struct its_vpe *vpe, unsigned long flags)
273 {
274 raw_spin_unlock_irqrestore(&vpe->vpe_lock, flags);
275 }
276
277 static struct irq_chip its_vpe_irq_chip;
278
irq_to_cpuid_lock(struct irq_data * d,unsigned long * flags)279 static int irq_to_cpuid_lock(struct irq_data *d, unsigned long *flags)
280 {
281 struct its_vpe *vpe = NULL;
282 int cpu;
283
284 if (d->chip == &its_vpe_irq_chip) {
285 vpe = irq_data_get_irq_chip_data(d);
286 } else {
287 struct its_vlpi_map *map = get_vlpi_map(d);
288 if (map)
289 vpe = map->vpe;
290 }
291
292 if (vpe) {
293 cpu = vpe_to_cpuid_lock(vpe, flags);
294 } else {
295 /* Physical LPIs are already locked via the irq_desc lock */
296 struct its_device *its_dev = irq_data_get_irq_chip_data(d);
297 cpu = its_dev->event_map.col_map[its_get_event_id(d)];
298 /* Keep GCC quiet... */
299 *flags = 0;
300 }
301
302 return cpu;
303 }
304
irq_to_cpuid_unlock(struct irq_data * d,unsigned long flags)305 static void irq_to_cpuid_unlock(struct irq_data *d, unsigned long flags)
306 {
307 struct its_vpe *vpe = NULL;
308
309 if (d->chip == &its_vpe_irq_chip) {
310 vpe = irq_data_get_irq_chip_data(d);
311 } else {
312 struct its_vlpi_map *map = get_vlpi_map(d);
313 if (map)
314 vpe = map->vpe;
315 }
316
317 if (vpe)
318 vpe_to_cpuid_unlock(vpe, flags);
319 }
320
valid_col(struct its_collection * col)321 static struct its_collection *valid_col(struct its_collection *col)
322 {
323 if (WARN_ON_ONCE(col->target_address & GENMASK_ULL(15, 0)))
324 return NULL;
325
326 return col;
327 }
328
valid_vpe(struct its_node * its,struct its_vpe * vpe)329 static struct its_vpe *valid_vpe(struct its_node *its, struct its_vpe *vpe)
330 {
331 if (valid_col(its->collections + vpe->col_idx))
332 return vpe;
333
334 return NULL;
335 }
336
337 /*
338 * ITS command descriptors - parameters to be encoded in a command
339 * block.
340 */
341 struct its_cmd_desc {
342 union {
343 struct {
344 struct its_device *dev;
345 u32 event_id;
346 } its_inv_cmd;
347
348 struct {
349 struct its_device *dev;
350 u32 event_id;
351 } its_clear_cmd;
352
353 struct {
354 struct its_device *dev;
355 u32 event_id;
356 } its_int_cmd;
357
358 struct {
359 struct its_device *dev;
360 int valid;
361 } its_mapd_cmd;
362
363 struct {
364 struct its_collection *col;
365 int valid;
366 } its_mapc_cmd;
367
368 struct {
369 struct its_device *dev;
370 u32 phys_id;
371 u32 event_id;
372 } its_mapti_cmd;
373
374 struct {
375 struct its_device *dev;
376 struct its_collection *col;
377 u32 event_id;
378 } its_movi_cmd;
379
380 struct {
381 struct its_device *dev;
382 u32 event_id;
383 } its_discard_cmd;
384
385 struct {
386 struct its_collection *col;
387 } its_invall_cmd;
388
389 struct {
390 struct its_vpe *vpe;
391 } its_vinvall_cmd;
392
393 struct {
394 struct its_vpe *vpe;
395 struct its_collection *col;
396 bool valid;
397 } its_vmapp_cmd;
398
399 struct {
400 struct its_vpe *vpe;
401 struct its_device *dev;
402 u32 virt_id;
403 u32 event_id;
404 bool db_enabled;
405 } its_vmapti_cmd;
406
407 struct {
408 struct its_vpe *vpe;
409 struct its_device *dev;
410 u32 event_id;
411 bool db_enabled;
412 } its_vmovi_cmd;
413
414 struct {
415 struct its_vpe *vpe;
416 struct its_collection *col;
417 u16 seq_num;
418 u16 its_list;
419 } its_vmovp_cmd;
420
421 struct {
422 struct its_vpe *vpe;
423 } its_invdb_cmd;
424
425 struct {
426 struct its_vpe *vpe;
427 u8 sgi;
428 u8 priority;
429 bool enable;
430 bool group;
431 bool clear;
432 } its_vsgi_cmd;
433 };
434 };
435
436 /*
437 * The ITS command block, which is what the ITS actually parses.
438 */
439 struct its_cmd_block {
440 union {
441 u64 raw_cmd[4];
442 __le64 raw_cmd_le[4];
443 };
444 };
445
446 #define ITS_CMD_QUEUE_SZ SZ_64K
447 #define ITS_CMD_QUEUE_NR_ENTRIES (ITS_CMD_QUEUE_SZ / sizeof(struct its_cmd_block))
448
449 typedef struct its_collection *(*its_cmd_builder_t)(struct its_node *,
450 struct its_cmd_block *,
451 struct its_cmd_desc *);
452
453 typedef struct its_vpe *(*its_cmd_vbuilder_t)(struct its_node *,
454 struct its_cmd_block *,
455 struct its_cmd_desc *);
456
457 static void its_mask_encode(u64 *raw_cmd, u64 val, int h, int l)
458 {
459 u64 mask = GENMASK_ULL(h, l);
460 *raw_cmd &= ~mask;
461 *raw_cmd |= (val << l) & mask;
462 }
463
its_encode_cmd(struct its_cmd_block * cmd,u8 cmd_nr)464 static void its_encode_cmd(struct its_cmd_block *cmd, u8 cmd_nr)
465 {
466 its_mask_encode(&cmd->raw_cmd[0], cmd_nr, 7, 0);
467 }
468
its_encode_devid(struct its_cmd_block * cmd,u32 devid)469 static void its_encode_devid(struct its_cmd_block *cmd, u32 devid)
470 {
471 its_mask_encode(&cmd->raw_cmd[0], devid, 63, 32);
472 }
473
its_encode_event_id(struct its_cmd_block * cmd,u32 id)474 static void its_encode_event_id(struct its_cmd_block *cmd, u32 id)
475 {
476 its_mask_encode(&cmd->raw_cmd[1], id, 31, 0);
477 }
478
its_encode_phys_id(struct its_cmd_block * cmd,u32 phys_id)479 static void its_encode_phys_id(struct its_cmd_block *cmd, u32 phys_id)
480 {
481 its_mask_encode(&cmd->raw_cmd[1], phys_id, 63, 32);
482 }
483
its_encode_size(struct its_cmd_block * cmd,u8 size)484 static void its_encode_size(struct its_cmd_block *cmd, u8 size)
485 {
486 its_mask_encode(&cmd->raw_cmd[1], size, 4, 0);
487 }
488
its_encode_itt(struct its_cmd_block * cmd,u64 itt_addr)489 static void its_encode_itt(struct its_cmd_block *cmd, u64 itt_addr)
490 {
491 its_mask_encode(&cmd->raw_cmd[2], itt_addr >> 8, 51, 8);
492 }
493
its_encode_valid(struct its_cmd_block * cmd,int valid)494 static void its_encode_valid(struct its_cmd_block *cmd, int valid)
495 {
496 its_mask_encode(&cmd->raw_cmd[2], !!valid, 63, 63);
497 }
498
its_encode_target(struct its_cmd_block * cmd,u64 target_addr)499 static void its_encode_target(struct its_cmd_block *cmd, u64 target_addr)
500 {
501 its_mask_encode(&cmd->raw_cmd[2], target_addr >> 16, 51, 16);
502 }
503
its_encode_collection(struct its_cmd_block * cmd,u16 col)504 static void its_encode_collection(struct its_cmd_block *cmd, u16 col)
505 {
506 its_mask_encode(&cmd->raw_cmd[2], col, 15, 0);
507 }
508
its_encode_vpeid(struct its_cmd_block * cmd,u16 vpeid)509 static void its_encode_vpeid(struct its_cmd_block *cmd, u16 vpeid)
510 {
511 its_mask_encode(&cmd->raw_cmd[1], vpeid, 47, 32);
512 }
513
its_encode_virt_id(struct its_cmd_block * cmd,u32 virt_id)514 static void its_encode_virt_id(struct its_cmd_block *cmd, u32 virt_id)
515 {
516 its_mask_encode(&cmd->raw_cmd[2], virt_id, 31, 0);
517 }
518
its_encode_db_phys_id(struct its_cmd_block * cmd,u32 db_phys_id)519 static void its_encode_db_phys_id(struct its_cmd_block *cmd, u32 db_phys_id)
520 {
521 its_mask_encode(&cmd->raw_cmd[2], db_phys_id, 63, 32);
522 }
523
its_encode_db_valid(struct its_cmd_block * cmd,bool db_valid)524 static void its_encode_db_valid(struct its_cmd_block *cmd, bool db_valid)
525 {
526 its_mask_encode(&cmd->raw_cmd[2], db_valid, 0, 0);
527 }
528
its_encode_seq_num(struct its_cmd_block * cmd,u16 seq_num)529 static void its_encode_seq_num(struct its_cmd_block *cmd, u16 seq_num)
530 {
531 its_mask_encode(&cmd->raw_cmd[0], seq_num, 47, 32);
532 }
533
its_encode_its_list(struct its_cmd_block * cmd,u16 its_list)534 static void its_encode_its_list(struct its_cmd_block *cmd, u16 its_list)
535 {
536 its_mask_encode(&cmd->raw_cmd[1], its_list, 15, 0);
537 }
538
its_encode_vpt_addr(struct its_cmd_block * cmd,u64 vpt_pa)539 static void its_encode_vpt_addr(struct its_cmd_block *cmd, u64 vpt_pa)
540 {
541 its_mask_encode(&cmd->raw_cmd[3], vpt_pa >> 16, 51, 16);
542 }
543
its_encode_vpt_size(struct its_cmd_block * cmd,u8 vpt_size)544 static void its_encode_vpt_size(struct its_cmd_block *cmd, u8 vpt_size)
545 {
546 its_mask_encode(&cmd->raw_cmd[3], vpt_size, 4, 0);
547 }
548
its_encode_vconf_addr(struct its_cmd_block * cmd,u64 vconf_pa)549 static void its_encode_vconf_addr(struct its_cmd_block *cmd, u64 vconf_pa)
550 {
551 its_mask_encode(&cmd->raw_cmd[0], vconf_pa >> 16, 51, 16);
552 }
553
its_encode_alloc(struct its_cmd_block * cmd,bool alloc)554 static void its_encode_alloc(struct its_cmd_block *cmd, bool alloc)
555 {
556 its_mask_encode(&cmd->raw_cmd[0], alloc, 8, 8);
557 }
558
its_encode_ptz(struct its_cmd_block * cmd,bool ptz)559 static void its_encode_ptz(struct its_cmd_block *cmd, bool ptz)
560 {
561 its_mask_encode(&cmd->raw_cmd[0], ptz, 9, 9);
562 }
563
its_encode_vmapp_default_db(struct its_cmd_block * cmd,u32 vpe_db_lpi)564 static void its_encode_vmapp_default_db(struct its_cmd_block *cmd,
565 u32 vpe_db_lpi)
566 {
567 its_mask_encode(&cmd->raw_cmd[1], vpe_db_lpi, 31, 0);
568 }
569
its_encode_vmovp_default_db(struct its_cmd_block * cmd,u32 vpe_db_lpi)570 static void its_encode_vmovp_default_db(struct its_cmd_block *cmd,
571 u32 vpe_db_lpi)
572 {
573 its_mask_encode(&cmd->raw_cmd[3], vpe_db_lpi, 31, 0);
574 }
575
its_encode_db(struct its_cmd_block * cmd,bool db)576 static void its_encode_db(struct its_cmd_block *cmd, bool db)
577 {
578 its_mask_encode(&cmd->raw_cmd[2], db, 63, 63);
579 }
580
its_encode_sgi_intid(struct its_cmd_block * cmd,u8 sgi)581 static void its_encode_sgi_intid(struct its_cmd_block *cmd, u8 sgi)
582 {
583 its_mask_encode(&cmd->raw_cmd[0], sgi, 35, 32);
584 }
585
its_encode_sgi_priority(struct its_cmd_block * cmd,u8 prio)586 static void its_encode_sgi_priority(struct its_cmd_block *cmd, u8 prio)
587 {
588 its_mask_encode(&cmd->raw_cmd[0], prio >> 4, 23, 20);
589 }
590
its_encode_sgi_group(struct its_cmd_block * cmd,bool grp)591 static void its_encode_sgi_group(struct its_cmd_block *cmd, bool grp)
592 {
593 its_mask_encode(&cmd->raw_cmd[0], grp, 10, 10);
594 }
595
its_encode_sgi_clear(struct its_cmd_block * cmd,bool clr)596 static void its_encode_sgi_clear(struct its_cmd_block *cmd, bool clr)
597 {
598 its_mask_encode(&cmd->raw_cmd[0], clr, 9, 9);
599 }
600
its_encode_sgi_enable(struct its_cmd_block * cmd,bool en)601 static void its_encode_sgi_enable(struct its_cmd_block *cmd, bool en)
602 {
603 its_mask_encode(&cmd->raw_cmd[0], en, 8, 8);
604 }
605
its_fixup_cmd(struct its_cmd_block * cmd)606 static inline void its_fixup_cmd(struct its_cmd_block *cmd)
607 {
608 /* Let's fixup BE commands */
609 cmd->raw_cmd_le[0] = cpu_to_le64(cmd->raw_cmd[0]);
610 cmd->raw_cmd_le[1] = cpu_to_le64(cmd->raw_cmd[1]);
611 cmd->raw_cmd_le[2] = cpu_to_le64(cmd->raw_cmd[2]);
612 cmd->raw_cmd_le[3] = cpu_to_le64(cmd->raw_cmd[3]);
613 }
614
its_build_mapd_cmd(struct its_node * its,struct its_cmd_block * cmd,struct its_cmd_desc * desc)615 static struct its_collection *its_build_mapd_cmd(struct its_node *its,
616 struct its_cmd_block *cmd,
617 struct its_cmd_desc *desc)
618 {
619 unsigned long itt_addr;
620 u8 size = ilog2(desc->its_mapd_cmd.dev->nr_ites);
621
622 itt_addr = virt_to_phys(desc->its_mapd_cmd.dev->itt);
623 itt_addr = ALIGN(itt_addr, ITS_ITT_ALIGN);
624
625 its_encode_cmd(cmd, GITS_CMD_MAPD);
626 its_encode_devid(cmd, desc->its_mapd_cmd.dev->device_id);
627 its_encode_size(cmd, size - 1);
628 its_encode_itt(cmd, itt_addr);
629 its_encode_valid(cmd, desc->its_mapd_cmd.valid);
630
631 its_fixup_cmd(cmd);
632
633 return NULL;
634 }
635
its_build_mapc_cmd(struct its_node * its,struct its_cmd_block * cmd,struct its_cmd_desc * desc)636 static struct its_collection *its_build_mapc_cmd(struct its_node *its,
637 struct its_cmd_block *cmd,
638 struct its_cmd_desc *desc)
639 {
640 its_encode_cmd(cmd, GITS_CMD_MAPC);
641 its_encode_collection(cmd, desc->its_mapc_cmd.col->col_id);
642 its_encode_target(cmd, desc->its_mapc_cmd.col->target_address);
643 its_encode_valid(cmd, desc->its_mapc_cmd.valid);
644
645 its_fixup_cmd(cmd);
646
647 return desc->its_mapc_cmd.col;
648 }
649
its_build_mapti_cmd(struct its_node * its,struct its_cmd_block * cmd,struct its_cmd_desc * desc)650 static struct its_collection *its_build_mapti_cmd(struct its_node *its,
651 struct its_cmd_block *cmd,
652 struct its_cmd_desc *desc)
653 {
654 struct its_collection *col;
655
656 col = dev_event_to_col(desc->its_mapti_cmd.dev,
657 desc->its_mapti_cmd.event_id);
658
659 its_encode_cmd(cmd, GITS_CMD_MAPTI);
660 its_encode_devid(cmd, desc->its_mapti_cmd.dev->device_id);
661 its_encode_event_id(cmd, desc->its_mapti_cmd.event_id);
662 its_encode_phys_id(cmd, desc->its_mapti_cmd.phys_id);
663 its_encode_collection(cmd, col->col_id);
664
665 its_fixup_cmd(cmd);
666
667 return valid_col(col);
668 }
669
its_build_movi_cmd(struct its_node * its,struct its_cmd_block * cmd,struct its_cmd_desc * desc)670 static struct its_collection *its_build_movi_cmd(struct its_node *its,
671 struct its_cmd_block *cmd,
672 struct its_cmd_desc *desc)
673 {
674 struct its_collection *col;
675
676 col = dev_event_to_col(desc->its_movi_cmd.dev,
677 desc->its_movi_cmd.event_id);
678
679 its_encode_cmd(cmd, GITS_CMD_MOVI);
680 its_encode_devid(cmd, desc->its_movi_cmd.dev->device_id);
681 its_encode_event_id(cmd, desc->its_movi_cmd.event_id);
682 its_encode_collection(cmd, desc->its_movi_cmd.col->col_id);
683
684 its_fixup_cmd(cmd);
685
686 return valid_col(col);
687 }
688
its_build_discard_cmd(struct its_node * its,struct its_cmd_block * cmd,struct its_cmd_desc * desc)689 static struct its_collection *its_build_discard_cmd(struct its_node *its,
690 struct its_cmd_block *cmd,
691 struct its_cmd_desc *desc)
692 {
693 struct its_collection *col;
694
695 col = dev_event_to_col(desc->its_discard_cmd.dev,
696 desc->its_discard_cmd.event_id);
697
698 its_encode_cmd(cmd, GITS_CMD_DISCARD);
699 its_encode_devid(cmd, desc->its_discard_cmd.dev->device_id);
700 its_encode_event_id(cmd, desc->its_discard_cmd.event_id);
701
702 its_fixup_cmd(cmd);
703
704 return valid_col(col);
705 }
706
its_build_inv_cmd(struct its_node * its,struct its_cmd_block * cmd,struct its_cmd_desc * desc)707 static struct its_collection *its_build_inv_cmd(struct its_node *its,
708 struct its_cmd_block *cmd,
709 struct its_cmd_desc *desc)
710 {
711 struct its_collection *col;
712
713 col = dev_event_to_col(desc->its_inv_cmd.dev,
714 desc->its_inv_cmd.event_id);
715
716 its_encode_cmd(cmd, GITS_CMD_INV);
717 its_encode_devid(cmd, desc->its_inv_cmd.dev->device_id);
718 its_encode_event_id(cmd, desc->its_inv_cmd.event_id);
719
720 its_fixup_cmd(cmd);
721
722 return valid_col(col);
723 }
724
its_build_int_cmd(struct its_node * its,struct its_cmd_block * cmd,struct its_cmd_desc * desc)725 static struct its_collection *its_build_int_cmd(struct its_node *its,
726 struct its_cmd_block *cmd,
727 struct its_cmd_desc *desc)
728 {
729 struct its_collection *col;
730
731 col = dev_event_to_col(desc->its_int_cmd.dev,
732 desc->its_int_cmd.event_id);
733
734 its_encode_cmd(cmd, GITS_CMD_INT);
735 its_encode_devid(cmd, desc->its_int_cmd.dev->device_id);
736 its_encode_event_id(cmd, desc->its_int_cmd.event_id);
737
738 its_fixup_cmd(cmd);
739
740 return valid_col(col);
741 }
742
its_build_clear_cmd(struct its_node * its,struct its_cmd_block * cmd,struct its_cmd_desc * desc)743 static struct its_collection *its_build_clear_cmd(struct its_node *its,
744 struct its_cmd_block *cmd,
745 struct its_cmd_desc *desc)
746 {
747 struct its_collection *col;
748
749 col = dev_event_to_col(desc->its_clear_cmd.dev,
750 desc->its_clear_cmd.event_id);
751
752 its_encode_cmd(cmd, GITS_CMD_CLEAR);
753 its_encode_devid(cmd, desc->its_clear_cmd.dev->device_id);
754 its_encode_event_id(cmd, desc->its_clear_cmd.event_id);
755
756 its_fixup_cmd(cmd);
757
758 return valid_col(col);
759 }
760
its_build_invall_cmd(struct its_node * its,struct its_cmd_block * cmd,struct its_cmd_desc * desc)761 static struct its_collection *its_build_invall_cmd(struct its_node *its,
762 struct its_cmd_block *cmd,
763 struct its_cmd_desc *desc)
764 {
765 its_encode_cmd(cmd, GITS_CMD_INVALL);
766 its_encode_collection(cmd, desc->its_invall_cmd.col->col_id);
767
768 its_fixup_cmd(cmd);
769
770 return desc->its_invall_cmd.col;
771 }
772
its_build_vinvall_cmd(struct its_node * its,struct its_cmd_block * cmd,struct its_cmd_desc * desc)773 static struct its_vpe *its_build_vinvall_cmd(struct its_node *its,
774 struct its_cmd_block *cmd,
775 struct its_cmd_desc *desc)
776 {
777 its_encode_cmd(cmd, GITS_CMD_VINVALL);
778 its_encode_vpeid(cmd, desc->its_vinvall_cmd.vpe->vpe_id);
779
780 its_fixup_cmd(cmd);
781
782 return valid_vpe(its, desc->its_vinvall_cmd.vpe);
783 }
784
its_build_vmapp_cmd(struct its_node * its,struct its_cmd_block * cmd,struct its_cmd_desc * desc)785 static struct its_vpe *its_build_vmapp_cmd(struct its_node *its,
786 struct its_cmd_block *cmd,
787 struct its_cmd_desc *desc)
788 {
789 struct its_vpe *vpe = valid_vpe(its, desc->its_vmapp_cmd.vpe);
790 unsigned long vpt_addr, vconf_addr;
791 u64 target;
792 bool alloc;
793
794 its_encode_cmd(cmd, GITS_CMD_VMAPP);
795 its_encode_vpeid(cmd, desc->its_vmapp_cmd.vpe->vpe_id);
796 its_encode_valid(cmd, desc->its_vmapp_cmd.valid);
797
798 if (!desc->its_vmapp_cmd.valid) {
799 alloc = !atomic_dec_return(&desc->its_vmapp_cmd.vpe->vmapp_count);
800 if (is_v4_1(its)) {
801 its_encode_alloc(cmd, alloc);
802 /*
803 * Unmapping a VPE is self-synchronizing on GICv4.1,
804 * no need to issue a VSYNC.
805 */
806 vpe = NULL;
807 }
808
809 goto out;
810 }
811
812 vpt_addr = virt_to_phys(page_address(desc->its_vmapp_cmd.vpe->vpt_page));
813 target = desc->its_vmapp_cmd.col->target_address + its->vlpi_redist_offset;
814
815 its_encode_target(cmd, target);
816 its_encode_vpt_addr(cmd, vpt_addr);
817 its_encode_vpt_size(cmd, LPI_NRBITS - 1);
818
819 alloc = !atomic_fetch_inc(&desc->its_vmapp_cmd.vpe->vmapp_count);
820
821 if (!is_v4_1(its))
822 goto out;
823
824 vconf_addr = virt_to_phys(page_address(desc->its_vmapp_cmd.vpe->its_vm->vprop_page));
825
826 its_encode_alloc(cmd, alloc);
827
828 /*
829 * GICv4.1 provides a way to get the VLPI state, which needs the vPE
830 * to be unmapped first, and in this case, we may remap the vPE
831 * back while the VPT is not empty. So we can't assume that the
832 * VPT is empty on map. This is why we never advertise PTZ.
833 */
834 its_encode_ptz(cmd, false);
835 its_encode_vconf_addr(cmd, vconf_addr);
836 its_encode_vmapp_default_db(cmd, desc->its_vmapp_cmd.vpe->vpe_db_lpi);
837
838 out:
839 its_fixup_cmd(cmd);
840
841 return vpe;
842 }
843
its_build_vmapti_cmd(struct its_node * its,struct its_cmd_block * cmd,struct its_cmd_desc * desc)844 static struct its_vpe *its_build_vmapti_cmd(struct its_node *its,
845 struct its_cmd_block *cmd,
846 struct its_cmd_desc *desc)
847 {
848 u32 db;
849
850 if (!is_v4_1(its) && desc->its_vmapti_cmd.db_enabled)
851 db = desc->its_vmapti_cmd.vpe->vpe_db_lpi;
852 else
853 db = 1023;
854
855 its_encode_cmd(cmd, GITS_CMD_VMAPTI);
856 its_encode_devid(cmd, desc->its_vmapti_cmd.dev->device_id);
857 its_encode_vpeid(cmd, desc->its_vmapti_cmd.vpe->vpe_id);
858 its_encode_event_id(cmd, desc->its_vmapti_cmd.event_id);
859 its_encode_db_phys_id(cmd, db);
860 its_encode_virt_id(cmd, desc->its_vmapti_cmd.virt_id);
861
862 its_fixup_cmd(cmd);
863
864 return valid_vpe(its, desc->its_vmapti_cmd.vpe);
865 }
866
its_build_vmovi_cmd(struct its_node * its,struct its_cmd_block * cmd,struct its_cmd_desc * desc)867 static struct its_vpe *its_build_vmovi_cmd(struct its_node *its,
868 struct its_cmd_block *cmd,
869 struct its_cmd_desc *desc)
870 {
871 u32 db;
872
873 if (!is_v4_1(its) && desc->its_vmovi_cmd.db_enabled)
874 db = desc->its_vmovi_cmd.vpe->vpe_db_lpi;
875 else
876 db = 1023;
877
878 its_encode_cmd(cmd, GITS_CMD_VMOVI);
879 its_encode_devid(cmd, desc->its_vmovi_cmd.dev->device_id);
880 its_encode_vpeid(cmd, desc->its_vmovi_cmd.vpe->vpe_id);
881 its_encode_event_id(cmd, desc->its_vmovi_cmd.event_id);
882 its_encode_db_phys_id(cmd, db);
883 its_encode_db_valid(cmd, true);
884
885 its_fixup_cmd(cmd);
886
887 return valid_vpe(its, desc->its_vmovi_cmd.vpe);
888 }
889
its_build_vmovp_cmd(struct its_node * its,struct its_cmd_block * cmd,struct its_cmd_desc * desc)890 static struct its_vpe *its_build_vmovp_cmd(struct its_node *its,
891 struct its_cmd_block *cmd,
892 struct its_cmd_desc *desc)
893 {
894 u64 target;
895
896 target = desc->its_vmovp_cmd.col->target_address + its->vlpi_redist_offset;
897 its_encode_cmd(cmd, GITS_CMD_VMOVP);
898 its_encode_seq_num(cmd, desc->its_vmovp_cmd.seq_num);
899 its_encode_its_list(cmd, desc->its_vmovp_cmd.its_list);
900 its_encode_vpeid(cmd, desc->its_vmovp_cmd.vpe->vpe_id);
901 its_encode_target(cmd, target);
902
903 if (is_v4_1(its)) {
904 its_encode_db(cmd, true);
905 its_encode_vmovp_default_db(cmd, desc->its_vmovp_cmd.vpe->vpe_db_lpi);
906 }
907
908 its_fixup_cmd(cmd);
909
910 return valid_vpe(its, desc->its_vmovp_cmd.vpe);
911 }
912
its_build_vinv_cmd(struct its_node * its,struct its_cmd_block * cmd,struct its_cmd_desc * desc)913 static struct its_vpe *its_build_vinv_cmd(struct its_node *its,
914 struct its_cmd_block *cmd,
915 struct its_cmd_desc *desc)
916 {
917 struct its_vlpi_map *map;
918
919 map = dev_event_to_vlpi_map(desc->its_inv_cmd.dev,
920 desc->its_inv_cmd.event_id);
921
922 its_encode_cmd(cmd, GITS_CMD_INV);
923 its_encode_devid(cmd, desc->its_inv_cmd.dev->device_id);
924 its_encode_event_id(cmd, desc->its_inv_cmd.event_id);
925
926 its_fixup_cmd(cmd);
927
928 return valid_vpe(its, map->vpe);
929 }
930
its_build_vint_cmd(struct its_node * its,struct its_cmd_block * cmd,struct its_cmd_desc * desc)931 static struct its_vpe *its_build_vint_cmd(struct its_node *its,
932 struct its_cmd_block *cmd,
933 struct its_cmd_desc *desc)
934 {
935 struct its_vlpi_map *map;
936
937 map = dev_event_to_vlpi_map(desc->its_int_cmd.dev,
938 desc->its_int_cmd.event_id);
939
940 its_encode_cmd(cmd, GITS_CMD_INT);
941 its_encode_devid(cmd, desc->its_int_cmd.dev->device_id);
942 its_encode_event_id(cmd, desc->its_int_cmd.event_id);
943
944 its_fixup_cmd(cmd);
945
946 return valid_vpe(its, map->vpe);
947 }
948
its_build_vclear_cmd(struct its_node * its,struct its_cmd_block * cmd,struct its_cmd_desc * desc)949 static struct its_vpe *its_build_vclear_cmd(struct its_node *its,
950 struct its_cmd_block *cmd,
951 struct its_cmd_desc *desc)
952 {
953 struct its_vlpi_map *map;
954
955 map = dev_event_to_vlpi_map(desc->its_clear_cmd.dev,
956 desc->its_clear_cmd.event_id);
957
958 its_encode_cmd(cmd, GITS_CMD_CLEAR);
959 its_encode_devid(cmd, desc->its_clear_cmd.dev->device_id);
960 its_encode_event_id(cmd, desc->its_clear_cmd.event_id);
961
962 its_fixup_cmd(cmd);
963
964 return valid_vpe(its, map->vpe);
965 }
966
its_build_invdb_cmd(struct its_node * its,struct its_cmd_block * cmd,struct its_cmd_desc * desc)967 static struct its_vpe *its_build_invdb_cmd(struct its_node *its,
968 struct its_cmd_block *cmd,
969 struct its_cmd_desc *desc)
970 {
971 if (WARN_ON(!is_v4_1(its)))
972 return NULL;
973
974 its_encode_cmd(cmd, GITS_CMD_INVDB);
975 its_encode_vpeid(cmd, desc->its_invdb_cmd.vpe->vpe_id);
976
977 its_fixup_cmd(cmd);
978
979 return valid_vpe(its, desc->its_invdb_cmd.vpe);
980 }
981
its_build_vsgi_cmd(struct its_node * its,struct its_cmd_block * cmd,struct its_cmd_desc * desc)982 static struct its_vpe *its_build_vsgi_cmd(struct its_node *its,
983 struct its_cmd_block *cmd,
984 struct its_cmd_desc *desc)
985 {
986 if (WARN_ON(!is_v4_1(its)))
987 return NULL;
988
989 its_encode_cmd(cmd, GITS_CMD_VSGI);
990 its_encode_vpeid(cmd, desc->its_vsgi_cmd.vpe->vpe_id);
991 its_encode_sgi_intid(cmd, desc->its_vsgi_cmd.sgi);
992 its_encode_sgi_priority(cmd, desc->its_vsgi_cmd.priority);
993 its_encode_sgi_group(cmd, desc->its_vsgi_cmd.group);
994 its_encode_sgi_clear(cmd, desc->its_vsgi_cmd.clear);
995 its_encode_sgi_enable(cmd, desc->its_vsgi_cmd.enable);
996
997 its_fixup_cmd(cmd);
998
999 return valid_vpe(its, desc->its_vsgi_cmd.vpe);
1000 }
1001
its_cmd_ptr_to_offset(struct its_node * its,struct its_cmd_block * ptr)1002 static u64 its_cmd_ptr_to_offset(struct its_node *its,
1003 struct its_cmd_block *ptr)
1004 {
1005 return (ptr - its->cmd_base) * sizeof(*ptr);
1006 }
1007
its_queue_full(struct its_node * its)1008 static int its_queue_full(struct its_node *its)
1009 {
1010 int widx;
1011 int ridx;
1012
1013 widx = its->cmd_write - its->cmd_base;
1014 ridx = readl_relaxed(its->base + GITS_CREADR) / sizeof(struct its_cmd_block);
1015
1016 /* This is incredibly unlikely to happen, unless the ITS locks up. */
1017 if (((widx + 1) % ITS_CMD_QUEUE_NR_ENTRIES) == ridx)
1018 return 1;
1019
1020 return 0;
1021 }
1022
its_allocate_entry(struct its_node * its)1023 static struct its_cmd_block *its_allocate_entry(struct its_node *its)
1024 {
1025 struct its_cmd_block *cmd;
1026 u32 count = 1000000; /* 1s! */
1027
1028 while (its_queue_full(its)) {
1029 count--;
1030 if (!count) {
1031 pr_err_ratelimited("ITS queue not draining\n");
1032 return NULL;
1033 }
1034 cpu_relax();
1035 udelay(1);
1036 }
1037
1038 cmd = its->cmd_write++;
1039
1040 /* Handle queue wrapping */
1041 if (its->cmd_write == (its->cmd_base + ITS_CMD_QUEUE_NR_ENTRIES))
1042 its->cmd_write = its->cmd_base;
1043
1044 /* Clear command */
1045 cmd->raw_cmd[0] = 0;
1046 cmd->raw_cmd[1] = 0;
1047 cmd->raw_cmd[2] = 0;
1048 cmd->raw_cmd[3] = 0;
1049
1050 return cmd;
1051 }
1052
its_post_commands(struct its_node * its)1053 static struct its_cmd_block *its_post_commands(struct its_node *its)
1054 {
1055 u64 wr = its_cmd_ptr_to_offset(its, its->cmd_write);
1056
1057 writel_relaxed(wr, its->base + GITS_CWRITER);
1058
1059 return its->cmd_write;
1060 }
1061
its_flush_cmd(struct its_node * its,struct its_cmd_block * cmd)1062 static void its_flush_cmd(struct its_node *its, struct its_cmd_block *cmd)
1063 {
1064 /*
1065 * Make sure the commands written to memory are observable by
1066 * the ITS.
1067 */
1068 if (its->flags & ITS_FLAGS_CMDQ_NEEDS_FLUSHING)
1069 gic_flush_dcache_to_poc(cmd, sizeof(*cmd));
1070 else
1071 dsb(ishst);
1072 }
1073
its_wait_for_range_completion(struct its_node * its,u64 prev_idx,struct its_cmd_block * to)1074 static int its_wait_for_range_completion(struct its_node *its,
1075 u64 prev_idx,
1076 struct its_cmd_block *to)
1077 {
1078 u64 rd_idx, to_idx, linear_idx;
1079 u32 count = 1000000; /* 1s! */
1080
1081 /* Linearize to_idx if the command set has wrapped around */
1082 to_idx = its_cmd_ptr_to_offset(its, to);
1083 if (to_idx < prev_idx)
1084 to_idx += ITS_CMD_QUEUE_SZ;
1085
1086 linear_idx = prev_idx;
1087
1088 while (1) {
1089 s64 delta;
1090
1091 rd_idx = readl_relaxed(its->base + GITS_CREADR);
1092
1093 /*
1094 * Compute the read pointer progress, taking the
1095 * potential wrap-around into account.
1096 */
1097 delta = rd_idx - prev_idx;
1098 if (rd_idx < prev_idx)
1099 delta += ITS_CMD_QUEUE_SZ;
1100
1101 linear_idx += delta;
1102 if (linear_idx >= to_idx)
1103 break;
1104
1105 count--;
1106 if (!count) {
1107 pr_err_ratelimited("ITS queue timeout (%llu %llu)\n",
1108 to_idx, linear_idx);
1109 return -1;
1110 }
1111 prev_idx = rd_idx;
1112 cpu_relax();
1113 udelay(1);
1114 }
1115
1116 return 0;
1117 }
1118
1119 /* Warning, macro hell follows */
1120 #define BUILD_SINGLE_CMD_FUNC(name, buildtype, synctype, buildfn) \
1121 void name(struct its_node *its, \
1122 buildtype builder, \
1123 struct its_cmd_desc *desc) \
1124 { \
1125 struct its_cmd_block *cmd, *sync_cmd, *next_cmd; \
1126 synctype *sync_obj; \
1127 unsigned long flags; \
1128 u64 rd_idx; \
1129 \
1130 raw_spin_lock_irqsave(&its->lock, flags); \
1131 \
1132 cmd = its_allocate_entry(its); \
1133 if (!cmd) { /* We're soooooo screewed... */ \
1134 raw_spin_unlock_irqrestore(&its->lock, flags); \
1135 return; \
1136 } \
1137 sync_obj = builder(its, cmd, desc); \
1138 its_flush_cmd(its, cmd); \
1139 \
1140 if (sync_obj) { \
1141 sync_cmd = its_allocate_entry(its); \
1142 if (!sync_cmd) \
1143 goto post; \
1144 \
1145 buildfn(its, sync_cmd, sync_obj); \
1146 its_flush_cmd(its, sync_cmd); \
1147 } \
1148 \
1149 post: \
1150 rd_idx = readl_relaxed(its->base + GITS_CREADR); \
1151 next_cmd = its_post_commands(its); \
1152 raw_spin_unlock_irqrestore(&its->lock, flags); \
1153 \
1154 if (its_wait_for_range_completion(its, rd_idx, next_cmd)) \
1155 pr_err_ratelimited("ITS cmd %ps failed\n", builder); \
1156 }
1157
its_build_sync_cmd(struct its_node * its,struct its_cmd_block * sync_cmd,struct its_collection * sync_col)1158 static void its_build_sync_cmd(struct its_node *its,
1159 struct its_cmd_block *sync_cmd,
1160 struct its_collection *sync_col)
1161 {
1162 its_encode_cmd(sync_cmd, GITS_CMD_SYNC);
1163 its_encode_target(sync_cmd, sync_col->target_address);
1164
1165 its_fixup_cmd(sync_cmd);
1166 }
1167
BUILD_SINGLE_CMD_FUNC(its_send_single_command,its_cmd_builder_t,struct its_collection,its_build_sync_cmd)1168 static BUILD_SINGLE_CMD_FUNC(its_send_single_command, its_cmd_builder_t,
1169 struct its_collection, its_build_sync_cmd)
1170
1171 static void its_build_vsync_cmd(struct its_node *its,
1172 struct its_cmd_block *sync_cmd,
1173 struct its_vpe *sync_vpe)
1174 {
1175 its_encode_cmd(sync_cmd, GITS_CMD_VSYNC);
1176 its_encode_vpeid(sync_cmd, sync_vpe->vpe_id);
1177
1178 its_fixup_cmd(sync_cmd);
1179 }
1180
BUILD_SINGLE_CMD_FUNC(its_send_single_vcommand,its_cmd_vbuilder_t,struct its_vpe,its_build_vsync_cmd)1181 static BUILD_SINGLE_CMD_FUNC(its_send_single_vcommand, its_cmd_vbuilder_t,
1182 struct its_vpe, its_build_vsync_cmd)
1183
1184 static void its_send_int(struct its_device *dev, u32 event_id)
1185 {
1186 struct its_cmd_desc desc;
1187
1188 desc.its_int_cmd.dev = dev;
1189 desc.its_int_cmd.event_id = event_id;
1190
1191 its_send_single_command(dev->its, its_build_int_cmd, &desc);
1192 }
1193
its_send_clear(struct its_device * dev,u32 event_id)1194 static void its_send_clear(struct its_device *dev, u32 event_id)
1195 {
1196 struct its_cmd_desc desc;
1197
1198 desc.its_clear_cmd.dev = dev;
1199 desc.its_clear_cmd.event_id = event_id;
1200
1201 its_send_single_command(dev->its, its_build_clear_cmd, &desc);
1202 }
1203
its_send_inv(struct its_device * dev,u32 event_id)1204 static void its_send_inv(struct its_device *dev, u32 event_id)
1205 {
1206 struct its_cmd_desc desc;
1207
1208 desc.its_inv_cmd.dev = dev;
1209 desc.its_inv_cmd.event_id = event_id;
1210
1211 its_send_single_command(dev->its, its_build_inv_cmd, &desc);
1212 }
1213
its_send_mapd(struct its_device * dev,int valid)1214 static void its_send_mapd(struct its_device *dev, int valid)
1215 {
1216 struct its_cmd_desc desc;
1217
1218 desc.its_mapd_cmd.dev = dev;
1219 desc.its_mapd_cmd.valid = !!valid;
1220
1221 its_send_single_command(dev->its, its_build_mapd_cmd, &desc);
1222 }
1223
its_send_mapc(struct its_node * its,struct its_collection * col,int valid)1224 static void its_send_mapc(struct its_node *its, struct its_collection *col,
1225 int valid)
1226 {
1227 struct its_cmd_desc desc;
1228
1229 desc.its_mapc_cmd.col = col;
1230 desc.its_mapc_cmd.valid = !!valid;
1231
1232 its_send_single_command(its, its_build_mapc_cmd, &desc);
1233 }
1234
its_send_mapti(struct its_device * dev,u32 irq_id,u32 id)1235 static void its_send_mapti(struct its_device *dev, u32 irq_id, u32 id)
1236 {
1237 struct its_cmd_desc desc;
1238
1239 desc.its_mapti_cmd.dev = dev;
1240 desc.its_mapti_cmd.phys_id = irq_id;
1241 desc.its_mapti_cmd.event_id = id;
1242
1243 its_send_single_command(dev->its, its_build_mapti_cmd, &desc);
1244 }
1245
its_send_movi(struct its_device * dev,struct its_collection * col,u32 id)1246 static void its_send_movi(struct its_device *dev,
1247 struct its_collection *col, u32 id)
1248 {
1249 struct its_cmd_desc desc;
1250
1251 desc.its_movi_cmd.dev = dev;
1252 desc.its_movi_cmd.col = col;
1253 desc.its_movi_cmd.event_id = id;
1254
1255 its_send_single_command(dev->its, its_build_movi_cmd, &desc);
1256 }
1257
its_send_discard(struct its_device * dev,u32 id)1258 static void its_send_discard(struct its_device *dev, u32 id)
1259 {
1260 struct its_cmd_desc desc;
1261
1262 desc.its_discard_cmd.dev = dev;
1263 desc.its_discard_cmd.event_id = id;
1264
1265 its_send_single_command(dev->its, its_build_discard_cmd, &desc);
1266 }
1267
its_send_invall(struct its_node * its,struct its_collection * col)1268 static void its_send_invall(struct its_node *its, struct its_collection *col)
1269 {
1270 struct its_cmd_desc desc;
1271
1272 desc.its_invall_cmd.col = col;
1273
1274 its_send_single_command(its, its_build_invall_cmd, &desc);
1275 }
1276
its_send_vmapti(struct its_device * dev,u32 id)1277 static void its_send_vmapti(struct its_device *dev, u32 id)
1278 {
1279 struct its_vlpi_map *map = dev_event_to_vlpi_map(dev, id);
1280 struct its_cmd_desc desc;
1281
1282 desc.its_vmapti_cmd.vpe = map->vpe;
1283 desc.its_vmapti_cmd.dev = dev;
1284 desc.its_vmapti_cmd.virt_id = map->vintid;
1285 desc.its_vmapti_cmd.event_id = id;
1286 desc.its_vmapti_cmd.db_enabled = map->db_enabled;
1287
1288 its_send_single_vcommand(dev->its, its_build_vmapti_cmd, &desc);
1289 }
1290
its_send_vmovi(struct its_device * dev,u32 id)1291 static void its_send_vmovi(struct its_device *dev, u32 id)
1292 {
1293 struct its_vlpi_map *map = dev_event_to_vlpi_map(dev, id);
1294 struct its_cmd_desc desc;
1295
1296 desc.its_vmovi_cmd.vpe = map->vpe;
1297 desc.its_vmovi_cmd.dev = dev;
1298 desc.its_vmovi_cmd.event_id = id;
1299 desc.its_vmovi_cmd.db_enabled = map->db_enabled;
1300
1301 its_send_single_vcommand(dev->its, its_build_vmovi_cmd, &desc);
1302 }
1303
its_send_vmapp(struct its_node * its,struct its_vpe * vpe,bool valid)1304 static void its_send_vmapp(struct its_node *its,
1305 struct its_vpe *vpe, bool valid)
1306 {
1307 struct its_cmd_desc desc;
1308
1309 desc.its_vmapp_cmd.vpe = vpe;
1310 desc.its_vmapp_cmd.valid = valid;
1311 desc.its_vmapp_cmd.col = &its->collections[vpe->col_idx];
1312
1313 its_send_single_vcommand(its, its_build_vmapp_cmd, &desc);
1314 }
1315
its_send_vmovp(struct its_vpe * vpe)1316 static void its_send_vmovp(struct its_vpe *vpe)
1317 {
1318 struct its_cmd_desc desc = {};
1319 struct its_node *its;
1320 unsigned long flags;
1321 int col_id = vpe->col_idx;
1322
1323 desc.its_vmovp_cmd.vpe = vpe;
1324
1325 if (!its_list_map) {
1326 its = list_first_entry(&its_nodes, struct its_node, entry);
1327 desc.its_vmovp_cmd.col = &its->collections[col_id];
1328 its_send_single_vcommand(its, its_build_vmovp_cmd, &desc);
1329 return;
1330 }
1331
1332 /*
1333 * Yet another marvel of the architecture. If using the
1334 * its_list "feature", we need to make sure that all ITSs
1335 * receive all VMOVP commands in the same order. The only way
1336 * to guarantee this is to make vmovp a serialization point.
1337 *
1338 * Wall <-- Head.
1339 */
1340 raw_spin_lock_irqsave(&vmovp_lock, flags);
1341
1342 desc.its_vmovp_cmd.seq_num = vmovp_seq_num++;
1343 desc.its_vmovp_cmd.its_list = get_its_list(vpe->its_vm);
1344
1345 /* Emit VMOVPs */
1346 list_for_each_entry(its, &its_nodes, entry) {
1347 if (!is_v4(its))
1348 continue;
1349
1350 if (!require_its_list_vmovp(vpe->its_vm, its))
1351 continue;
1352
1353 desc.its_vmovp_cmd.col = &its->collections[col_id];
1354 its_send_single_vcommand(its, its_build_vmovp_cmd, &desc);
1355 }
1356
1357 raw_spin_unlock_irqrestore(&vmovp_lock, flags);
1358 }
1359
its_send_vinvall(struct its_node * its,struct its_vpe * vpe)1360 static void its_send_vinvall(struct its_node *its, struct its_vpe *vpe)
1361 {
1362 struct its_cmd_desc desc;
1363
1364 desc.its_vinvall_cmd.vpe = vpe;
1365 its_send_single_vcommand(its, its_build_vinvall_cmd, &desc);
1366 }
1367
its_send_vinv(struct its_device * dev,u32 event_id)1368 static void its_send_vinv(struct its_device *dev, u32 event_id)
1369 {
1370 struct its_cmd_desc desc;
1371
1372 /*
1373 * There is no real VINV command. This is just a normal INV,
1374 * with a VSYNC instead of a SYNC.
1375 */
1376 desc.its_inv_cmd.dev = dev;
1377 desc.its_inv_cmd.event_id = event_id;
1378
1379 its_send_single_vcommand(dev->its, its_build_vinv_cmd, &desc);
1380 }
1381
its_send_vint(struct its_device * dev,u32 event_id)1382 static void its_send_vint(struct its_device *dev, u32 event_id)
1383 {
1384 struct its_cmd_desc desc;
1385
1386 /*
1387 * There is no real VINT command. This is just a normal INT,
1388 * with a VSYNC instead of a SYNC.
1389 */
1390 desc.its_int_cmd.dev = dev;
1391 desc.its_int_cmd.event_id = event_id;
1392
1393 its_send_single_vcommand(dev->its, its_build_vint_cmd, &desc);
1394 }
1395
its_send_vclear(struct its_device * dev,u32 event_id)1396 static void its_send_vclear(struct its_device *dev, u32 event_id)
1397 {
1398 struct its_cmd_desc desc;
1399
1400 /*
1401 * There is no real VCLEAR command. This is just a normal CLEAR,
1402 * with a VSYNC instead of a SYNC.
1403 */
1404 desc.its_clear_cmd.dev = dev;
1405 desc.its_clear_cmd.event_id = event_id;
1406
1407 its_send_single_vcommand(dev->its, its_build_vclear_cmd, &desc);
1408 }
1409
its_send_invdb(struct its_node * its,struct its_vpe * vpe)1410 static void its_send_invdb(struct its_node *its, struct its_vpe *vpe)
1411 {
1412 struct its_cmd_desc desc;
1413
1414 desc.its_invdb_cmd.vpe = vpe;
1415 its_send_single_vcommand(its, its_build_invdb_cmd, &desc);
1416 }
1417
1418 /*
1419 * irqchip functions - assumes MSI, mostly.
1420 */
lpi_write_config(struct irq_data * d,u8 clr,u8 set)1421 static void lpi_write_config(struct irq_data *d, u8 clr, u8 set)
1422 {
1423 struct its_vlpi_map *map = get_vlpi_map(d);
1424 irq_hw_number_t hwirq;
1425 void *va;
1426 u8 *cfg;
1427
1428 if (map) {
1429 va = page_address(map->vm->vprop_page);
1430 hwirq = map->vintid;
1431
1432 /* Remember the updated property */
1433 map->properties &= ~clr;
1434 map->properties |= set | LPI_PROP_GROUP1;
1435 } else {
1436 va = gic_rdists->prop_table_va;
1437 hwirq = d->hwirq;
1438 }
1439
1440 cfg = va + hwirq - 8192;
1441 *cfg &= ~clr;
1442 *cfg |= set | LPI_PROP_GROUP1;
1443
1444 /*
1445 * Make the above write visible to the redistributors.
1446 * And yes, we're flushing exactly: One. Single. Byte.
1447 * Humpf...
1448 */
1449 if (gic_rdists->flags & RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING)
1450 gic_flush_dcache_to_poc(cfg, sizeof(*cfg));
1451 else
1452 dsb(ishst);
1453 }
1454
wait_for_syncr(void __iomem * rdbase)1455 static void wait_for_syncr(void __iomem *rdbase)
1456 {
1457 while (readl_relaxed(rdbase + GICR_SYNCR) & 1)
1458 cpu_relax();
1459 }
1460
__direct_lpi_inv(struct irq_data * d,u64 val)1461 static void __direct_lpi_inv(struct irq_data *d, u64 val)
1462 {
1463 void __iomem *rdbase;
1464 unsigned long flags;
1465 int cpu;
1466
1467 /* Target the redistributor this LPI is currently routed to */
1468 cpu = irq_to_cpuid_lock(d, &flags);
1469 raw_spin_lock(&gic_data_rdist_cpu(cpu)->rd_lock);
1470
1471 rdbase = per_cpu_ptr(gic_rdists->rdist, cpu)->rd_base;
1472 gic_write_lpir(val, rdbase + GICR_INVLPIR);
1473 wait_for_syncr(rdbase);
1474
1475 raw_spin_unlock(&gic_data_rdist_cpu(cpu)->rd_lock);
1476 irq_to_cpuid_unlock(d, flags);
1477 }
1478
direct_lpi_inv(struct irq_data * d)1479 static void direct_lpi_inv(struct irq_data *d)
1480 {
1481 struct its_vlpi_map *map = get_vlpi_map(d);
1482 u64 val;
1483
1484 if (map) {
1485 struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1486
1487 WARN_ON(!is_v4_1(its_dev->its));
1488
1489 val = GICR_INVLPIR_V;
1490 val |= FIELD_PREP(GICR_INVLPIR_VPEID, map->vpe->vpe_id);
1491 val |= FIELD_PREP(GICR_INVLPIR_INTID, map->vintid);
1492 } else {
1493 val = d->hwirq;
1494 }
1495
1496 __direct_lpi_inv(d, val);
1497 }
1498
lpi_update_config(struct irq_data * d,u8 clr,u8 set)1499 static void lpi_update_config(struct irq_data *d, u8 clr, u8 set)
1500 {
1501 struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1502
1503 lpi_write_config(d, clr, set);
1504 if (gic_rdists->has_direct_lpi &&
1505 (is_v4_1(its_dev->its) || !irqd_is_forwarded_to_vcpu(d)))
1506 direct_lpi_inv(d);
1507 else if (!irqd_is_forwarded_to_vcpu(d))
1508 its_send_inv(its_dev, its_get_event_id(d));
1509 else
1510 its_send_vinv(its_dev, its_get_event_id(d));
1511 }
1512
its_vlpi_set_doorbell(struct irq_data * d,bool enable)1513 static void its_vlpi_set_doorbell(struct irq_data *d, bool enable)
1514 {
1515 struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1516 u32 event = its_get_event_id(d);
1517 struct its_vlpi_map *map;
1518
1519 /*
1520 * GICv4.1 does away with the per-LPI nonsense, nothing to do
1521 * here.
1522 */
1523 if (is_v4_1(its_dev->its))
1524 return;
1525
1526 map = dev_event_to_vlpi_map(its_dev, event);
1527
1528 if (map->db_enabled == enable)
1529 return;
1530
1531 map->db_enabled = enable;
1532
1533 /*
1534 * More fun with the architecture:
1535 *
1536 * Ideally, we'd issue a VMAPTI to set the doorbell to its LPI
1537 * value or to 1023, depending on the enable bit. But that
1538 * would be issuing a mapping for an /existing/ DevID+EventID
1539 * pair, which is UNPREDICTABLE. Instead, let's issue a VMOVI
1540 * to the /same/ vPE, using this opportunity to adjust the
1541 * doorbell. Mouahahahaha. We loves it, Precious.
1542 */
1543 its_send_vmovi(its_dev, event);
1544 }
1545
its_mask_irq(struct irq_data * d)1546 static void its_mask_irq(struct irq_data *d)
1547 {
1548 if (irqd_is_forwarded_to_vcpu(d))
1549 its_vlpi_set_doorbell(d, false);
1550
1551 lpi_update_config(d, LPI_PROP_ENABLED, 0);
1552 }
1553
its_unmask_irq(struct irq_data * d)1554 static void its_unmask_irq(struct irq_data *d)
1555 {
1556 if (irqd_is_forwarded_to_vcpu(d))
1557 its_vlpi_set_doorbell(d, true);
1558
1559 lpi_update_config(d, 0, LPI_PROP_ENABLED);
1560 }
1561
its_read_lpi_count(struct irq_data * d,int cpu)1562 static __maybe_unused u32 its_read_lpi_count(struct irq_data *d, int cpu)
1563 {
1564 if (irqd_affinity_is_managed(d))
1565 return atomic_read(&per_cpu_ptr(&cpu_lpi_count, cpu)->managed);
1566
1567 return atomic_read(&per_cpu_ptr(&cpu_lpi_count, cpu)->unmanaged);
1568 }
1569
its_inc_lpi_count(struct irq_data * d,int cpu)1570 static void its_inc_lpi_count(struct irq_data *d, int cpu)
1571 {
1572 if (irqd_affinity_is_managed(d))
1573 atomic_inc(&per_cpu_ptr(&cpu_lpi_count, cpu)->managed);
1574 else
1575 atomic_inc(&per_cpu_ptr(&cpu_lpi_count, cpu)->unmanaged);
1576 }
1577
its_dec_lpi_count(struct irq_data * d,int cpu)1578 static void its_dec_lpi_count(struct irq_data *d, int cpu)
1579 {
1580 if (irqd_affinity_is_managed(d))
1581 atomic_dec(&per_cpu_ptr(&cpu_lpi_count, cpu)->managed);
1582 else
1583 atomic_dec(&per_cpu_ptr(&cpu_lpi_count, cpu)->unmanaged);
1584 }
1585
cpumask_pick_least_loaded(struct irq_data * d,const struct cpumask * cpu_mask)1586 static unsigned int cpumask_pick_least_loaded(struct irq_data *d,
1587 const struct cpumask *cpu_mask)
1588 {
1589 unsigned int cpu = nr_cpu_ids, tmp;
1590 int count = S32_MAX;
1591
1592 for_each_cpu(tmp, cpu_mask) {
1593 int this_count = its_read_lpi_count(d, tmp);
1594 if (this_count < count) {
1595 cpu = tmp;
1596 count = this_count;
1597 }
1598 }
1599
1600 return cpu;
1601 }
1602
1603 /*
1604 * As suggested by Thomas Gleixner in:
1605 * https://lore.kernel.org/r/87h80q2aoc.fsf@nanos.tec.linutronix.de
1606 */
its_select_cpu(struct irq_data * d,const struct cpumask * aff_mask)1607 static int its_select_cpu(struct irq_data *d,
1608 const struct cpumask *aff_mask)
1609 {
1610 struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1611 static DEFINE_RAW_SPINLOCK(tmpmask_lock);
1612 static struct cpumask __tmpmask;
1613 struct cpumask *tmpmask;
1614 unsigned long flags;
1615 int cpu, node;
1616 node = its_dev->its->numa_node;
1617 tmpmask = &__tmpmask;
1618
1619 raw_spin_lock_irqsave(&tmpmask_lock, flags);
1620
1621 if (!irqd_affinity_is_managed(d)) {
1622 /* First try the NUMA node */
1623 if (node != NUMA_NO_NODE) {
1624 /*
1625 * Try the intersection of the affinity mask and the
1626 * node mask (and the online mask, just to be safe).
1627 */
1628 cpumask_and(tmpmask, cpumask_of_node(node), aff_mask);
1629 cpumask_and(tmpmask, tmpmask, cpu_online_mask);
1630
1631 /*
1632 * Ideally, we would check if the mask is empty, and
1633 * try again on the full node here.
1634 *
1635 * But it turns out that the way ACPI describes the
1636 * affinity for ITSs only deals about memory, and
1637 * not target CPUs, so it cannot describe a single
1638 * ITS placed next to two NUMA nodes.
1639 *
1640 * Instead, just fallback on the online mask. This
1641 * diverges from Thomas' suggestion above.
1642 */
1643 cpu = cpumask_pick_least_loaded(d, tmpmask);
1644 if (cpu < nr_cpu_ids)
1645 goto out;
1646
1647 /* If we can't cross sockets, give up */
1648 if ((its_dev->its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144))
1649 goto out;
1650
1651 /* If the above failed, expand the search */
1652 }
1653
1654 /* Try the intersection of the affinity and online masks */
1655 cpumask_and(tmpmask, aff_mask, cpu_online_mask);
1656
1657 /* If that doesn't fly, the online mask is the last resort */
1658 if (cpumask_empty(tmpmask))
1659 cpumask_copy(tmpmask, cpu_online_mask);
1660
1661 cpu = cpumask_pick_least_loaded(d, tmpmask);
1662 } else {
1663 cpumask_copy(tmpmask, aff_mask);
1664
1665 /* If we cannot cross sockets, limit the search to that node */
1666 if ((its_dev->its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144) &&
1667 node != NUMA_NO_NODE)
1668 cpumask_and(tmpmask, tmpmask, cpumask_of_node(node));
1669
1670 cpu = cpumask_pick_least_loaded(d, tmpmask);
1671 }
1672 out:
1673 raw_spin_unlock_irqrestore(&tmpmask_lock, flags);
1674
1675 pr_debug("IRQ%d -> %*pbl CPU%d\n", d->irq, cpumask_pr_args(aff_mask), cpu);
1676 return cpu;
1677 }
1678
its_set_affinity(struct irq_data * d,const struct cpumask * mask_val,bool force)1679 static int its_set_affinity(struct irq_data *d, const struct cpumask *mask_val,
1680 bool force)
1681 {
1682 struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1683 struct its_collection *target_col;
1684 u32 id = its_get_event_id(d);
1685 int cpu, prev_cpu;
1686
1687 /* A forwarded interrupt should use irq_set_vcpu_affinity */
1688 if (irqd_is_forwarded_to_vcpu(d))
1689 return -EINVAL;
1690
1691 prev_cpu = its_dev->event_map.col_map[id];
1692 its_dec_lpi_count(d, prev_cpu);
1693
1694 if (!force)
1695 cpu = its_select_cpu(d, mask_val);
1696 else
1697 cpu = cpumask_pick_least_loaded(d, mask_val);
1698
1699 if (cpu < 0 || cpu >= nr_cpu_ids)
1700 goto err;
1701
1702 /* don't set the affinity when the target cpu is same as current one */
1703 if (cpu != prev_cpu) {
1704 target_col = &its_dev->its->collections[cpu];
1705 its_send_movi(its_dev, target_col, id);
1706 its_dev->event_map.col_map[id] = cpu;
1707 irq_data_update_effective_affinity(d, cpumask_of(cpu));
1708 }
1709
1710 its_inc_lpi_count(d, cpu);
1711
1712 return IRQ_SET_MASK_OK_DONE;
1713
1714 err:
1715 its_inc_lpi_count(d, prev_cpu);
1716 return -EINVAL;
1717 }
1718
its_irq_get_msi_base(struct its_device * its_dev)1719 static u64 its_irq_get_msi_base(struct its_device *its_dev)
1720 {
1721 struct its_node *its = its_dev->its;
1722
1723 return its->phys_base + GITS_TRANSLATER;
1724 }
1725
its_irq_compose_msi_msg(struct irq_data * d,struct msi_msg * msg)1726 static void its_irq_compose_msi_msg(struct irq_data *d, struct msi_msg *msg)
1727 {
1728 struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1729 struct its_node *its;
1730 u64 addr;
1731
1732 its = its_dev->its;
1733 addr = its->get_msi_base(its_dev);
1734
1735 msg->address_lo = lower_32_bits(addr);
1736 msg->address_hi = upper_32_bits(addr);
1737 msg->data = its_get_event_id(d);
1738
1739 iommu_dma_compose_msi_msg(irq_data_get_msi_desc(d), msg);
1740 }
1741
its_irq_set_irqchip_state(struct irq_data * d,enum irqchip_irq_state which,bool state)1742 static int its_irq_set_irqchip_state(struct irq_data *d,
1743 enum irqchip_irq_state which,
1744 bool state)
1745 {
1746 struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1747 u32 event = its_get_event_id(d);
1748
1749 if (which != IRQCHIP_STATE_PENDING)
1750 return -EINVAL;
1751
1752 if (irqd_is_forwarded_to_vcpu(d)) {
1753 if (state)
1754 its_send_vint(its_dev, event);
1755 else
1756 its_send_vclear(its_dev, event);
1757 } else {
1758 if (state)
1759 its_send_int(its_dev, event);
1760 else
1761 its_send_clear(its_dev, event);
1762 }
1763
1764 return 0;
1765 }
1766
its_irq_retrigger(struct irq_data * d)1767 static int its_irq_retrigger(struct irq_data *d)
1768 {
1769 return !its_irq_set_irqchip_state(d, IRQCHIP_STATE_PENDING, true);
1770 }
1771
1772 /*
1773 * Two favourable cases:
1774 *
1775 * (a) Either we have a GICv4.1, and all vPEs have to be mapped at all times
1776 * for vSGI delivery
1777 *
1778 * (b) Or the ITSs do not use a list map, meaning that VMOVP is cheap enough
1779 * and we're better off mapping all VPEs always
1780 *
1781 * If neither (a) nor (b) is true, then we map vPEs on demand.
1782 *
1783 */
gic_requires_eager_mapping(void)1784 static bool gic_requires_eager_mapping(void)
1785 {
1786 if (!its_list_map || gic_rdists->has_rvpeid)
1787 return true;
1788
1789 return false;
1790 }
1791
its_map_vm(struct its_node * its,struct its_vm * vm)1792 static void its_map_vm(struct its_node *its, struct its_vm *vm)
1793 {
1794 unsigned long flags;
1795
1796 if (gic_requires_eager_mapping())
1797 return;
1798
1799 raw_spin_lock_irqsave(&vmovp_lock, flags);
1800
1801 /*
1802 * If the VM wasn't mapped yet, iterate over the vpes and get
1803 * them mapped now.
1804 */
1805 vm->vlpi_count[its->list_nr]++;
1806
1807 if (vm->vlpi_count[its->list_nr] == 1) {
1808 int i;
1809
1810 for (i = 0; i < vm->nr_vpes; i++) {
1811 struct its_vpe *vpe = vm->vpes[i];
1812 struct irq_data *d = irq_get_irq_data(vpe->irq);
1813
1814 /* Map the VPE to the first possible CPU */
1815 vpe->col_idx = cpumask_first(cpu_online_mask);
1816 its_send_vmapp(its, vpe, true);
1817 its_send_vinvall(its, vpe);
1818 irq_data_update_effective_affinity(d, cpumask_of(vpe->col_idx));
1819 }
1820 }
1821
1822 raw_spin_unlock_irqrestore(&vmovp_lock, flags);
1823 }
1824
its_unmap_vm(struct its_node * its,struct its_vm * vm)1825 static void its_unmap_vm(struct its_node *its, struct its_vm *vm)
1826 {
1827 unsigned long flags;
1828
1829 /* Not using the ITS list? Everything is always mapped. */
1830 if (gic_requires_eager_mapping())
1831 return;
1832
1833 raw_spin_lock_irqsave(&vmovp_lock, flags);
1834
1835 if (!--vm->vlpi_count[its->list_nr]) {
1836 int i;
1837
1838 for (i = 0; i < vm->nr_vpes; i++)
1839 its_send_vmapp(its, vm->vpes[i], false);
1840 }
1841
1842 raw_spin_unlock_irqrestore(&vmovp_lock, flags);
1843 }
1844
its_vlpi_map(struct irq_data * d,struct its_cmd_info * info)1845 static int its_vlpi_map(struct irq_data *d, struct its_cmd_info *info)
1846 {
1847 struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1848 u32 event = its_get_event_id(d);
1849
1850 if (!info->map)
1851 return -EINVAL;
1852
1853 if (!its_dev->event_map.vm) {
1854 struct its_vlpi_map *maps;
1855
1856 maps = kcalloc(its_dev->event_map.nr_lpis, sizeof(*maps),
1857 GFP_ATOMIC);
1858 if (!maps)
1859 return -ENOMEM;
1860
1861 its_dev->event_map.vm = info->map->vm;
1862 its_dev->event_map.vlpi_maps = maps;
1863 } else if (its_dev->event_map.vm != info->map->vm) {
1864 return -EINVAL;
1865 }
1866
1867 /* Get our private copy of the mapping information */
1868 its_dev->event_map.vlpi_maps[event] = *info->map;
1869
1870 if (irqd_is_forwarded_to_vcpu(d)) {
1871 /* Already mapped, move it around */
1872 its_send_vmovi(its_dev, event);
1873 } else {
1874 /* Ensure all the VPEs are mapped on this ITS */
1875 its_map_vm(its_dev->its, info->map->vm);
1876
1877 /*
1878 * Flag the interrupt as forwarded so that we can
1879 * start poking the virtual property table.
1880 */
1881 irqd_set_forwarded_to_vcpu(d);
1882
1883 /* Write out the property to the prop table */
1884 lpi_write_config(d, 0xff, info->map->properties);
1885
1886 /* Drop the physical mapping */
1887 its_send_discard(its_dev, event);
1888
1889 /* and install the virtual one */
1890 its_send_vmapti(its_dev, event);
1891
1892 /* Increment the number of VLPIs */
1893 its_dev->event_map.nr_vlpis++;
1894 }
1895
1896 return 0;
1897 }
1898
its_vlpi_get(struct irq_data * d,struct its_cmd_info * info)1899 static int its_vlpi_get(struct irq_data *d, struct its_cmd_info *info)
1900 {
1901 struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1902 struct its_vlpi_map *map;
1903
1904 map = get_vlpi_map(d);
1905
1906 if (!its_dev->event_map.vm || !map)
1907 return -EINVAL;
1908
1909 /* Copy our mapping information to the incoming request */
1910 *info->map = *map;
1911
1912 return 0;
1913 }
1914
its_vlpi_unmap(struct irq_data * d)1915 static int its_vlpi_unmap(struct irq_data *d)
1916 {
1917 struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1918 u32 event = its_get_event_id(d);
1919
1920 if (!its_dev->event_map.vm || !irqd_is_forwarded_to_vcpu(d))
1921 return -EINVAL;
1922
1923 /* Drop the virtual mapping */
1924 its_send_discard(its_dev, event);
1925
1926 /* and restore the physical one */
1927 irqd_clr_forwarded_to_vcpu(d);
1928 its_send_mapti(its_dev, d->hwirq, event);
1929 lpi_update_config(d, 0xff, (LPI_PROP_DEFAULT_PRIO |
1930 LPI_PROP_ENABLED |
1931 LPI_PROP_GROUP1));
1932
1933 /* Potentially unmap the VM from this ITS */
1934 its_unmap_vm(its_dev->its, its_dev->event_map.vm);
1935
1936 /*
1937 * Drop the refcount and make the device available again if
1938 * this was the last VLPI.
1939 */
1940 if (!--its_dev->event_map.nr_vlpis) {
1941 its_dev->event_map.vm = NULL;
1942 kfree(its_dev->event_map.vlpi_maps);
1943 }
1944
1945 return 0;
1946 }
1947
its_vlpi_prop_update(struct irq_data * d,struct its_cmd_info * info)1948 static int its_vlpi_prop_update(struct irq_data *d, struct its_cmd_info *info)
1949 {
1950 struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1951
1952 if (!its_dev->event_map.vm || !irqd_is_forwarded_to_vcpu(d))
1953 return -EINVAL;
1954
1955 if (info->cmd_type == PROP_UPDATE_AND_INV_VLPI)
1956 lpi_update_config(d, 0xff, info->config);
1957 else
1958 lpi_write_config(d, 0xff, info->config);
1959 its_vlpi_set_doorbell(d, !!(info->config & LPI_PROP_ENABLED));
1960
1961 return 0;
1962 }
1963
its_irq_set_vcpu_affinity(struct irq_data * d,void * vcpu_info)1964 static int its_irq_set_vcpu_affinity(struct irq_data *d, void *vcpu_info)
1965 {
1966 struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1967 struct its_cmd_info *info = vcpu_info;
1968
1969 /* Need a v4 ITS */
1970 if (!is_v4(its_dev->its))
1971 return -EINVAL;
1972
1973 guard(raw_spinlock_irq)(&its_dev->event_map.vlpi_lock);
1974
1975 /* Unmap request? */
1976 if (!info)
1977 return its_vlpi_unmap(d);
1978
1979 switch (info->cmd_type) {
1980 case MAP_VLPI:
1981 return its_vlpi_map(d, info);
1982
1983 case GET_VLPI:
1984 return its_vlpi_get(d, info);
1985
1986 case PROP_UPDATE_VLPI:
1987 case PROP_UPDATE_AND_INV_VLPI:
1988 return its_vlpi_prop_update(d, info);
1989
1990 default:
1991 return -EINVAL;
1992 }
1993 }
1994
1995 static struct irq_chip its_irq_chip = {
1996 .name = "ITS",
1997 .irq_mask = its_mask_irq,
1998 .irq_unmask = its_unmask_irq,
1999 .irq_eoi = irq_chip_eoi_parent,
2000 .irq_set_affinity = its_set_affinity,
2001 .irq_compose_msi_msg = its_irq_compose_msi_msg,
2002 .irq_set_irqchip_state = its_irq_set_irqchip_state,
2003 .irq_retrigger = its_irq_retrigger,
2004 .irq_set_vcpu_affinity = its_irq_set_vcpu_affinity,
2005 };
2006
2007
2008 /*
2009 * How we allocate LPIs:
2010 *
2011 * lpi_range_list contains ranges of LPIs that are to available to
2012 * allocate from. To allocate LPIs, just pick the first range that
2013 * fits the required allocation, and reduce it by the required
2014 * amount. Once empty, remove the range from the list.
2015 *
2016 * To free a range of LPIs, add a free range to the list, sort it and
2017 * merge the result if the new range happens to be adjacent to an
2018 * already free block.
2019 *
2020 * The consequence of the above is that allocation is cost is low, but
2021 * freeing is expensive. We assumes that freeing rarely occurs.
2022 */
2023 #define ITS_MAX_LPI_NRBITS 16 /* 64K LPIs */
2024
2025 static DEFINE_MUTEX(lpi_range_lock);
2026 static LIST_HEAD(lpi_range_list);
2027
2028 struct lpi_range {
2029 struct list_head entry;
2030 u32 base_id;
2031 u32 span;
2032 };
2033
mk_lpi_range(u32 base,u32 span)2034 static struct lpi_range *mk_lpi_range(u32 base, u32 span)
2035 {
2036 struct lpi_range *range;
2037
2038 range = kmalloc(sizeof(*range), GFP_KERNEL);
2039 if (range) {
2040 range->base_id = base;
2041 range->span = span;
2042 }
2043
2044 return range;
2045 }
2046
alloc_lpi_range(u32 nr_lpis,u32 * base)2047 static int alloc_lpi_range(u32 nr_lpis, u32 *base)
2048 {
2049 struct lpi_range *range, *tmp;
2050 int err = -ENOSPC;
2051
2052 mutex_lock(&lpi_range_lock);
2053
2054 list_for_each_entry_safe(range, tmp, &lpi_range_list, entry) {
2055 if (range->span >= nr_lpis) {
2056 *base = range->base_id;
2057 range->base_id += nr_lpis;
2058 range->span -= nr_lpis;
2059
2060 if (range->span == 0) {
2061 list_del(&range->entry);
2062 kfree(range);
2063 }
2064
2065 err = 0;
2066 break;
2067 }
2068 }
2069
2070 mutex_unlock(&lpi_range_lock);
2071
2072 pr_debug("ITS: alloc %u:%u\n", *base, nr_lpis);
2073 return err;
2074 }
2075
merge_lpi_ranges(struct lpi_range * a,struct lpi_range * b)2076 static void merge_lpi_ranges(struct lpi_range *a, struct lpi_range *b)
2077 {
2078 if (&a->entry == &lpi_range_list || &b->entry == &lpi_range_list)
2079 return;
2080 if (a->base_id + a->span != b->base_id)
2081 return;
2082 b->base_id = a->base_id;
2083 b->span += a->span;
2084 list_del(&a->entry);
2085 kfree(a);
2086 }
2087
free_lpi_range(u32 base,u32 nr_lpis)2088 static int free_lpi_range(u32 base, u32 nr_lpis)
2089 {
2090 struct lpi_range *new, *old;
2091
2092 new = mk_lpi_range(base, nr_lpis);
2093 if (!new)
2094 return -ENOMEM;
2095
2096 mutex_lock(&lpi_range_lock);
2097
2098 list_for_each_entry_reverse(old, &lpi_range_list, entry) {
2099 if (old->base_id < base)
2100 break;
2101 }
2102 /*
2103 * old is the last element with ->base_id smaller than base,
2104 * so new goes right after it. If there are no elements with
2105 * ->base_id smaller than base, &old->entry ends up pointing
2106 * at the head of the list, and inserting new it the start of
2107 * the list is the right thing to do in that case as well.
2108 */
2109 list_add(&new->entry, &old->entry);
2110 /*
2111 * Now check if we can merge with the preceding and/or
2112 * following ranges.
2113 */
2114 merge_lpi_ranges(old, new);
2115 merge_lpi_ranges(new, list_next_entry(new, entry));
2116
2117 mutex_unlock(&lpi_range_lock);
2118 return 0;
2119 }
2120
its_lpi_init(u32 id_bits)2121 static int __init its_lpi_init(u32 id_bits)
2122 {
2123 u32 lpis = (1UL << id_bits) - 8192;
2124 u32 numlpis;
2125 int err;
2126
2127 numlpis = 1UL << GICD_TYPER_NUM_LPIS(gic_rdists->gicd_typer);
2128
2129 if (numlpis > 2 && !WARN_ON(numlpis > lpis)) {
2130 lpis = numlpis;
2131 pr_info("ITS: Using hypervisor restricted LPI range [%u]\n",
2132 lpis);
2133 }
2134
2135 /*
2136 * Initializing the allocator is just the same as freeing the
2137 * full range of LPIs.
2138 */
2139 err = free_lpi_range(8192, lpis);
2140 pr_debug("ITS: Allocator initialized for %u LPIs\n", lpis);
2141 return err;
2142 }
2143
its_lpi_alloc(int nr_irqs,u32 * base,int * nr_ids)2144 static unsigned long *its_lpi_alloc(int nr_irqs, u32 *base, int *nr_ids)
2145 {
2146 unsigned long *bitmap = NULL;
2147 int err = 0;
2148
2149 do {
2150 err = alloc_lpi_range(nr_irqs, base);
2151 if (!err)
2152 break;
2153
2154 nr_irqs /= 2;
2155 } while (nr_irqs > 0);
2156
2157 if (!nr_irqs)
2158 err = -ENOSPC;
2159
2160 if (err)
2161 goto out;
2162
2163 bitmap = bitmap_zalloc(nr_irqs, GFP_ATOMIC);
2164 if (!bitmap)
2165 goto out;
2166
2167 *nr_ids = nr_irqs;
2168
2169 out:
2170 if (!bitmap)
2171 *base = *nr_ids = 0;
2172
2173 return bitmap;
2174 }
2175
its_lpi_free(unsigned long * bitmap,u32 base,u32 nr_ids)2176 static void its_lpi_free(unsigned long *bitmap, u32 base, u32 nr_ids)
2177 {
2178 WARN_ON(free_lpi_range(base, nr_ids));
2179 bitmap_free(bitmap);
2180 }
2181
gic_reset_prop_table(void * va)2182 static void gic_reset_prop_table(void *va)
2183 {
2184 /* Priority 0xa0, Group-1, disabled */
2185 memset(va, LPI_PROP_DEFAULT_PRIO | LPI_PROP_GROUP1, LPI_PROPBASE_SZ);
2186
2187 /* Make sure the GIC will observe the written configuration */
2188 gic_flush_dcache_to_poc(va, LPI_PROPBASE_SZ);
2189 }
2190
its_allocate_prop_table(gfp_t gfp_flags)2191 static struct page *its_allocate_prop_table(gfp_t gfp_flags)
2192 {
2193 struct page *prop_page;
2194
2195 prop_page = alloc_pages(gfp_flags, get_order(LPI_PROPBASE_SZ));
2196 if (!prop_page)
2197 return NULL;
2198
2199 gic_reset_prop_table(page_address(prop_page));
2200
2201 return prop_page;
2202 }
2203
its_free_prop_table(struct page * prop_page)2204 static void its_free_prop_table(struct page *prop_page)
2205 {
2206 free_pages((unsigned long)page_address(prop_page),
2207 get_order(LPI_PROPBASE_SZ));
2208 }
2209
gic_check_reserved_range(phys_addr_t addr,unsigned long size)2210 static bool gic_check_reserved_range(phys_addr_t addr, unsigned long size)
2211 {
2212 phys_addr_t start, end, addr_end;
2213 u64 i;
2214
2215 /*
2216 * We don't bother checking for a kdump kernel as by
2217 * construction, the LPI tables are out of this kernel's
2218 * memory map.
2219 */
2220 if (is_kdump_kernel())
2221 return true;
2222
2223 addr_end = addr + size - 1;
2224
2225 for_each_reserved_mem_range(i, &start, &end) {
2226 if (addr >= start && addr_end <= end)
2227 return true;
2228 }
2229
2230 /* Not found, not a good sign... */
2231 pr_warn("GICv3: Expected reserved range [%pa:%pa], not found\n",
2232 &addr, &addr_end);
2233 add_taint(TAINT_CRAP, LOCKDEP_STILL_OK);
2234 return false;
2235 }
2236
gic_reserve_range(phys_addr_t addr,unsigned long size)2237 static int gic_reserve_range(phys_addr_t addr, unsigned long size)
2238 {
2239 if (efi_enabled(EFI_CONFIG_TABLES))
2240 return efi_mem_reserve_persistent(addr, size);
2241
2242 return 0;
2243 }
2244
its_setup_lpi_prop_table(void)2245 static int __init its_setup_lpi_prop_table(void)
2246 {
2247 if (gic_rdists->flags & RDIST_FLAGS_RD_TABLES_PREALLOCATED) {
2248 u64 val;
2249
2250 val = gicr_read_propbaser(gic_data_rdist_rd_base() + GICR_PROPBASER);
2251 lpi_id_bits = (val & GICR_PROPBASER_IDBITS_MASK) + 1;
2252
2253 gic_rdists->prop_table_pa = val & GENMASK_ULL(51, 12);
2254 gic_rdists->prop_table_va = memremap(gic_rdists->prop_table_pa,
2255 LPI_PROPBASE_SZ,
2256 MEMREMAP_WB);
2257 gic_reset_prop_table(gic_rdists->prop_table_va);
2258 } else {
2259 struct page *page;
2260
2261 lpi_id_bits = min_t(u32,
2262 GICD_TYPER_ID_BITS(gic_rdists->gicd_typer),
2263 ITS_MAX_LPI_NRBITS);
2264 page = its_allocate_prop_table(GFP_NOWAIT);
2265 if (!page) {
2266 pr_err("Failed to allocate PROPBASE\n");
2267 return -ENOMEM;
2268 }
2269
2270 gic_rdists->prop_table_pa = page_to_phys(page);
2271 gic_rdists->prop_table_va = page_address(page);
2272 WARN_ON(gic_reserve_range(gic_rdists->prop_table_pa,
2273 LPI_PROPBASE_SZ));
2274 }
2275
2276 pr_info("GICv3: using LPI property table @%pa\n",
2277 &gic_rdists->prop_table_pa);
2278
2279 return its_lpi_init(lpi_id_bits);
2280 }
2281
2282 static const char *its_base_type_string[] = {
2283 [GITS_BASER_TYPE_DEVICE] = "Devices",
2284 [GITS_BASER_TYPE_VCPU] = "Virtual CPUs",
2285 [GITS_BASER_TYPE_RESERVED3] = "Reserved (3)",
2286 [GITS_BASER_TYPE_COLLECTION] = "Interrupt Collections",
2287 [GITS_BASER_TYPE_RESERVED5] = "Reserved (5)",
2288 [GITS_BASER_TYPE_RESERVED6] = "Reserved (6)",
2289 [GITS_BASER_TYPE_RESERVED7] = "Reserved (7)",
2290 };
2291
its_read_baser(struct its_node * its,struct its_baser * baser)2292 static u64 its_read_baser(struct its_node *its, struct its_baser *baser)
2293 {
2294 u32 idx = baser - its->tables;
2295
2296 return gits_read_baser(its->base + GITS_BASER + (idx << 3));
2297 }
2298
its_write_baser(struct its_node * its,struct its_baser * baser,u64 val)2299 static void its_write_baser(struct its_node *its, struct its_baser *baser,
2300 u64 val)
2301 {
2302 u32 idx = baser - its->tables;
2303
2304 gits_write_baser(val, its->base + GITS_BASER + (idx << 3));
2305 baser->val = its_read_baser(its, baser);
2306 }
2307
its_setup_baser(struct its_node * its,struct its_baser * baser,u64 cache,u64 shr,u32 order,bool indirect)2308 static int its_setup_baser(struct its_node *its, struct its_baser *baser,
2309 u64 cache, u64 shr, u32 order, bool indirect)
2310 {
2311 u64 val = its_read_baser(its, baser);
2312 u64 esz = GITS_BASER_ENTRY_SIZE(val);
2313 u64 type = GITS_BASER_TYPE(val);
2314 u64 baser_phys, tmp;
2315 u32 alloc_pages, psz;
2316 struct page *page;
2317 void *base;
2318
2319 psz = baser->psz;
2320 alloc_pages = (PAGE_ORDER_TO_SIZE(order) / psz);
2321 if (alloc_pages > GITS_BASER_PAGES_MAX) {
2322 pr_warn("ITS@%pa: %s too large, reduce ITS pages %u->%u\n",
2323 &its->phys_base, its_base_type_string[type],
2324 alloc_pages, GITS_BASER_PAGES_MAX);
2325 alloc_pages = GITS_BASER_PAGES_MAX;
2326 order = get_order(GITS_BASER_PAGES_MAX * psz);
2327 }
2328
2329 page = alloc_pages_node(its->numa_node, GFP_KERNEL | __GFP_ZERO, order);
2330 if (!page)
2331 return -ENOMEM;
2332
2333 base = (void *)page_address(page);
2334 baser_phys = virt_to_phys(base);
2335
2336 /* Check if the physical address of the memory is above 48bits */
2337 if (IS_ENABLED(CONFIG_ARM64_64K_PAGES) && (baser_phys >> 48)) {
2338
2339 /* 52bit PA is supported only when PageSize=64K */
2340 if (psz != SZ_64K) {
2341 pr_err("ITS: no 52bit PA support when psz=%d\n", psz);
2342 free_pages((unsigned long)base, order);
2343 return -ENXIO;
2344 }
2345
2346 /* Convert 52bit PA to 48bit field */
2347 baser_phys = GITS_BASER_PHYS_52_to_48(baser_phys);
2348 }
2349
2350 retry_baser:
2351 val = (baser_phys |
2352 (type << GITS_BASER_TYPE_SHIFT) |
2353 ((esz - 1) << GITS_BASER_ENTRY_SIZE_SHIFT) |
2354 ((alloc_pages - 1) << GITS_BASER_PAGES_SHIFT) |
2355 cache |
2356 shr |
2357 GITS_BASER_VALID);
2358
2359 val |= indirect ? GITS_BASER_INDIRECT : 0x0;
2360
2361 switch (psz) {
2362 case SZ_4K:
2363 val |= GITS_BASER_PAGE_SIZE_4K;
2364 break;
2365 case SZ_16K:
2366 val |= GITS_BASER_PAGE_SIZE_16K;
2367 break;
2368 case SZ_64K:
2369 val |= GITS_BASER_PAGE_SIZE_64K;
2370 break;
2371 }
2372
2373 if (!shr)
2374 gic_flush_dcache_to_poc(base, PAGE_ORDER_TO_SIZE(order));
2375
2376 its_write_baser(its, baser, val);
2377 tmp = baser->val;
2378
2379 if ((val ^ tmp) & GITS_BASER_SHAREABILITY_MASK) {
2380 /*
2381 * Shareability didn't stick. Just use
2382 * whatever the read reported, which is likely
2383 * to be the only thing this redistributor
2384 * supports. If that's zero, make it
2385 * non-cacheable as well.
2386 */
2387 shr = tmp & GITS_BASER_SHAREABILITY_MASK;
2388 if (!shr)
2389 cache = GITS_BASER_nC;
2390
2391 goto retry_baser;
2392 }
2393
2394 if (val != tmp) {
2395 pr_err("ITS@%pa: %s doesn't stick: %llx %llx\n",
2396 &its->phys_base, its_base_type_string[type],
2397 val, tmp);
2398 free_pages((unsigned long)base, order);
2399 return -ENXIO;
2400 }
2401
2402 baser->order = order;
2403 baser->base = base;
2404 baser->psz = psz;
2405 tmp = indirect ? GITS_LVL1_ENTRY_SIZE : esz;
2406
2407 pr_info("ITS@%pa: allocated %d %s @%lx (%s, esz %d, psz %dK, shr %d)\n",
2408 &its->phys_base, (int)(PAGE_ORDER_TO_SIZE(order) / (int)tmp),
2409 its_base_type_string[type],
2410 (unsigned long)virt_to_phys(base),
2411 indirect ? "indirect" : "flat", (int)esz,
2412 psz / SZ_1K, (int)shr >> GITS_BASER_SHAREABILITY_SHIFT);
2413
2414 return 0;
2415 }
2416
its_parse_indirect_baser(struct its_node * its,struct its_baser * baser,u32 * order,u32 ids)2417 static bool its_parse_indirect_baser(struct its_node *its,
2418 struct its_baser *baser,
2419 u32 *order, u32 ids)
2420 {
2421 u64 tmp = its_read_baser(its, baser);
2422 u64 type = GITS_BASER_TYPE(tmp);
2423 u64 esz = GITS_BASER_ENTRY_SIZE(tmp);
2424 u64 val = GITS_BASER_InnerShareable | GITS_BASER_RaWaWb;
2425 u32 new_order = *order;
2426 u32 psz = baser->psz;
2427 bool indirect = false;
2428
2429 /* No need to enable Indirection if memory requirement < (psz*2)bytes */
2430 if ((esz << ids) > (psz * 2)) {
2431 /*
2432 * Find out whether hw supports a single or two-level table by
2433 * table by reading bit at offset '62' after writing '1' to it.
2434 */
2435 its_write_baser(its, baser, val | GITS_BASER_INDIRECT);
2436 indirect = !!(baser->val & GITS_BASER_INDIRECT);
2437
2438 if (indirect) {
2439 /*
2440 * The size of the lvl2 table is equal to ITS page size
2441 * which is 'psz'. For computing lvl1 table size,
2442 * subtract ID bits that sparse lvl2 table from 'ids'
2443 * which is reported by ITS hardware times lvl1 table
2444 * entry size.
2445 */
2446 ids -= ilog2(psz / (int)esz);
2447 esz = GITS_LVL1_ENTRY_SIZE;
2448 }
2449 }
2450
2451 /*
2452 * Allocate as many entries as required to fit the
2453 * range of device IDs that the ITS can grok... The ID
2454 * space being incredibly sparse, this results in a
2455 * massive waste of memory if two-level device table
2456 * feature is not supported by hardware.
2457 */
2458 new_order = max_t(u32, get_order(esz << ids), new_order);
2459 if (new_order > MAX_ORDER) {
2460 new_order = MAX_ORDER;
2461 ids = ilog2(PAGE_ORDER_TO_SIZE(new_order) / (int)esz);
2462 pr_warn("ITS@%pa: %s Table too large, reduce ids %llu->%u\n",
2463 &its->phys_base, its_base_type_string[type],
2464 device_ids(its), ids);
2465 }
2466
2467 *order = new_order;
2468
2469 return indirect;
2470 }
2471
compute_common_aff(u64 val)2472 static u32 compute_common_aff(u64 val)
2473 {
2474 u32 aff, clpiaff;
2475
2476 aff = FIELD_GET(GICR_TYPER_AFFINITY, val);
2477 clpiaff = FIELD_GET(GICR_TYPER_COMMON_LPI_AFF, val);
2478
2479 return aff & ~(GENMASK(31, 0) >> (clpiaff * 8));
2480 }
2481
compute_its_aff(struct its_node * its)2482 static u32 compute_its_aff(struct its_node *its)
2483 {
2484 u64 val;
2485 u32 svpet;
2486
2487 /*
2488 * Reencode the ITS SVPET and MPIDR as a GICR_TYPER, and compute
2489 * the resulting affinity. We then use that to see if this match
2490 * our own affinity.
2491 */
2492 svpet = FIELD_GET(GITS_TYPER_SVPET, its->typer);
2493 val = FIELD_PREP(GICR_TYPER_COMMON_LPI_AFF, svpet);
2494 val |= FIELD_PREP(GICR_TYPER_AFFINITY, its->mpidr);
2495 return compute_common_aff(val);
2496 }
2497
find_sibling_its(struct its_node * cur_its)2498 static struct its_node *find_sibling_its(struct its_node *cur_its)
2499 {
2500 struct its_node *its;
2501 u32 aff;
2502
2503 if (!FIELD_GET(GITS_TYPER_SVPET, cur_its->typer))
2504 return NULL;
2505
2506 aff = compute_its_aff(cur_its);
2507
2508 list_for_each_entry(its, &its_nodes, entry) {
2509 u64 baser;
2510
2511 if (!is_v4_1(its) || its == cur_its)
2512 continue;
2513
2514 if (!FIELD_GET(GITS_TYPER_SVPET, its->typer))
2515 continue;
2516
2517 if (aff != compute_its_aff(its))
2518 continue;
2519
2520 /* GICv4.1 guarantees that the vPE table is GITS_BASER2 */
2521 baser = its->tables[2].val;
2522 if (!(baser & GITS_BASER_VALID))
2523 continue;
2524
2525 return its;
2526 }
2527
2528 return NULL;
2529 }
2530
its_free_tables(struct its_node * its)2531 static void its_free_tables(struct its_node *its)
2532 {
2533 int i;
2534
2535 for (i = 0; i < GITS_BASER_NR_REGS; i++) {
2536 if (its->tables[i].base) {
2537 free_pages((unsigned long)its->tables[i].base,
2538 its->tables[i].order);
2539 its->tables[i].base = NULL;
2540 }
2541 }
2542 }
2543
its_probe_baser_psz(struct its_node * its,struct its_baser * baser)2544 static int its_probe_baser_psz(struct its_node *its, struct its_baser *baser)
2545 {
2546 u64 psz = SZ_64K;
2547
2548 while (psz) {
2549 u64 val, gpsz;
2550
2551 val = its_read_baser(its, baser);
2552 val &= ~GITS_BASER_PAGE_SIZE_MASK;
2553
2554 switch (psz) {
2555 case SZ_64K:
2556 gpsz = GITS_BASER_PAGE_SIZE_64K;
2557 break;
2558 case SZ_16K:
2559 gpsz = GITS_BASER_PAGE_SIZE_16K;
2560 break;
2561 case SZ_4K:
2562 default:
2563 gpsz = GITS_BASER_PAGE_SIZE_4K;
2564 break;
2565 }
2566
2567 gpsz >>= GITS_BASER_PAGE_SIZE_SHIFT;
2568
2569 val |= FIELD_PREP(GITS_BASER_PAGE_SIZE_MASK, gpsz);
2570 its_write_baser(its, baser, val);
2571
2572 if (FIELD_GET(GITS_BASER_PAGE_SIZE_MASK, baser->val) == gpsz)
2573 break;
2574
2575 switch (psz) {
2576 case SZ_64K:
2577 psz = SZ_16K;
2578 break;
2579 case SZ_16K:
2580 psz = SZ_4K;
2581 break;
2582 case SZ_4K:
2583 default:
2584 return -1;
2585 }
2586 }
2587
2588 baser->psz = psz;
2589 return 0;
2590 }
2591
its_alloc_tables(struct its_node * its)2592 static int its_alloc_tables(struct its_node *its)
2593 {
2594 u64 shr = GITS_BASER_InnerShareable;
2595 u64 cache = GITS_BASER_RaWaWb;
2596 int err, i;
2597
2598 if (its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_22375)
2599 /* erratum 24313: ignore memory access type */
2600 cache = GITS_BASER_nCnB;
2601
2602 if (its->flags & ITS_FLAGS_FORCE_NON_SHAREABLE) {
2603 cache = GITS_BASER_nC;
2604 shr = 0;
2605 }
2606
2607 for (i = 0; i < GITS_BASER_NR_REGS; i++) {
2608 struct its_baser *baser = its->tables + i;
2609 u64 val = its_read_baser(its, baser);
2610 u64 type = GITS_BASER_TYPE(val);
2611 bool indirect = false;
2612 u32 order;
2613
2614 if (type == GITS_BASER_TYPE_NONE)
2615 continue;
2616
2617 if (its_probe_baser_psz(its, baser)) {
2618 its_free_tables(its);
2619 return -ENXIO;
2620 }
2621
2622 order = get_order(baser->psz);
2623
2624 switch (type) {
2625 case GITS_BASER_TYPE_DEVICE:
2626 indirect = its_parse_indirect_baser(its, baser, &order,
2627 device_ids(its));
2628 break;
2629
2630 case GITS_BASER_TYPE_VCPU:
2631 if (is_v4_1(its)) {
2632 struct its_node *sibling;
2633
2634 WARN_ON(i != 2);
2635 if ((sibling = find_sibling_its(its))) {
2636 *baser = sibling->tables[2];
2637 its_write_baser(its, baser, baser->val);
2638 continue;
2639 }
2640 }
2641
2642 indirect = its_parse_indirect_baser(its, baser, &order,
2643 ITS_MAX_VPEID_BITS);
2644 break;
2645 }
2646
2647 err = its_setup_baser(its, baser, cache, shr, order, indirect);
2648 if (err < 0) {
2649 its_free_tables(its);
2650 return err;
2651 }
2652
2653 /* Update settings which will be used for next BASERn */
2654 cache = baser->val & GITS_BASER_CACHEABILITY_MASK;
2655 shr = baser->val & GITS_BASER_SHAREABILITY_MASK;
2656 }
2657
2658 return 0;
2659 }
2660
inherit_vpe_l1_table_from_its(void)2661 static u64 inherit_vpe_l1_table_from_its(void)
2662 {
2663 struct its_node *its;
2664 u64 val;
2665 u32 aff;
2666
2667 val = gic_read_typer(gic_data_rdist_rd_base() + GICR_TYPER);
2668 aff = compute_common_aff(val);
2669
2670 list_for_each_entry(its, &its_nodes, entry) {
2671 u64 baser, addr;
2672
2673 if (!is_v4_1(its))
2674 continue;
2675
2676 if (!FIELD_GET(GITS_TYPER_SVPET, its->typer))
2677 continue;
2678
2679 if (aff != compute_its_aff(its))
2680 continue;
2681
2682 /* GICv4.1 guarantees that the vPE table is GITS_BASER2 */
2683 baser = its->tables[2].val;
2684 if (!(baser & GITS_BASER_VALID))
2685 continue;
2686
2687 /* We have a winner! */
2688 gic_data_rdist()->vpe_l1_base = its->tables[2].base;
2689
2690 val = GICR_VPROPBASER_4_1_VALID;
2691 if (baser & GITS_BASER_INDIRECT)
2692 val |= GICR_VPROPBASER_4_1_INDIRECT;
2693 val |= FIELD_PREP(GICR_VPROPBASER_4_1_PAGE_SIZE,
2694 FIELD_GET(GITS_BASER_PAGE_SIZE_MASK, baser));
2695 switch (FIELD_GET(GITS_BASER_PAGE_SIZE_MASK, baser)) {
2696 case GIC_PAGE_SIZE_64K:
2697 addr = GITS_BASER_ADDR_48_to_52(baser);
2698 break;
2699 default:
2700 addr = baser & GENMASK_ULL(47, 12);
2701 break;
2702 }
2703 val |= FIELD_PREP(GICR_VPROPBASER_4_1_ADDR, addr >> 12);
2704 if (rdists_support_shareable()) {
2705 val |= FIELD_PREP(GICR_VPROPBASER_SHAREABILITY_MASK,
2706 FIELD_GET(GITS_BASER_SHAREABILITY_MASK, baser));
2707 val |= FIELD_PREP(GICR_VPROPBASER_INNER_CACHEABILITY_MASK,
2708 FIELD_GET(GITS_BASER_INNER_CACHEABILITY_MASK, baser));
2709 }
2710 val |= FIELD_PREP(GICR_VPROPBASER_4_1_SIZE, GITS_BASER_NR_PAGES(baser) - 1);
2711
2712 return val;
2713 }
2714
2715 return 0;
2716 }
2717
inherit_vpe_l1_table_from_rd(cpumask_t ** mask)2718 static u64 inherit_vpe_l1_table_from_rd(cpumask_t **mask)
2719 {
2720 u32 aff;
2721 u64 val;
2722 int cpu;
2723
2724 val = gic_read_typer(gic_data_rdist_rd_base() + GICR_TYPER);
2725 aff = compute_common_aff(val);
2726
2727 for_each_possible_cpu(cpu) {
2728 void __iomem *base = gic_data_rdist_cpu(cpu)->rd_base;
2729
2730 if (!base || cpu == smp_processor_id())
2731 continue;
2732
2733 val = gic_read_typer(base + GICR_TYPER);
2734 if (aff != compute_common_aff(val))
2735 continue;
2736
2737 /*
2738 * At this point, we have a victim. This particular CPU
2739 * has already booted, and has an affinity that matches
2740 * ours wrt CommonLPIAff. Let's use its own VPROPBASER.
2741 * Make sure we don't write the Z bit in that case.
2742 */
2743 val = gicr_read_vpropbaser(base + SZ_128K + GICR_VPROPBASER);
2744 val &= ~GICR_VPROPBASER_4_1_Z;
2745
2746 gic_data_rdist()->vpe_l1_base = gic_data_rdist_cpu(cpu)->vpe_l1_base;
2747 *mask = gic_data_rdist_cpu(cpu)->vpe_table_mask;
2748
2749 return val;
2750 }
2751
2752 return 0;
2753 }
2754
allocate_vpe_l2_table(int cpu,u32 id)2755 static bool allocate_vpe_l2_table(int cpu, u32 id)
2756 {
2757 void __iomem *base = gic_data_rdist_cpu(cpu)->rd_base;
2758 unsigned int psz, esz, idx, npg, gpsz;
2759 u64 val;
2760 struct page *page;
2761 __le64 *table;
2762
2763 if (!gic_rdists->has_rvpeid)
2764 return true;
2765
2766 /* Skip non-present CPUs */
2767 if (!base)
2768 return true;
2769
2770 val = gicr_read_vpropbaser(base + SZ_128K + GICR_VPROPBASER);
2771
2772 esz = FIELD_GET(GICR_VPROPBASER_4_1_ENTRY_SIZE, val) + 1;
2773 gpsz = FIELD_GET(GICR_VPROPBASER_4_1_PAGE_SIZE, val);
2774 npg = FIELD_GET(GICR_VPROPBASER_4_1_SIZE, val) + 1;
2775
2776 switch (gpsz) {
2777 default:
2778 WARN_ON(1);
2779 fallthrough;
2780 case GIC_PAGE_SIZE_4K:
2781 psz = SZ_4K;
2782 break;
2783 case GIC_PAGE_SIZE_16K:
2784 psz = SZ_16K;
2785 break;
2786 case GIC_PAGE_SIZE_64K:
2787 psz = SZ_64K;
2788 break;
2789 }
2790
2791 /* Don't allow vpe_id that exceeds single, flat table limit */
2792 if (!(val & GICR_VPROPBASER_4_1_INDIRECT))
2793 return (id < (npg * psz / (esz * SZ_8)));
2794
2795 /* Compute 1st level table index & check if that exceeds table limit */
2796 idx = id >> ilog2(psz / (esz * SZ_8));
2797 if (idx >= (npg * psz / GITS_LVL1_ENTRY_SIZE))
2798 return false;
2799
2800 table = gic_data_rdist_cpu(cpu)->vpe_l1_base;
2801
2802 /* Allocate memory for 2nd level table */
2803 if (!table[idx]) {
2804 page = alloc_pages(GFP_KERNEL | __GFP_ZERO, get_order(psz));
2805 if (!page)
2806 return false;
2807
2808 /* Flush Lvl2 table to PoC if hw doesn't support coherency */
2809 if (!(val & GICR_VPROPBASER_SHAREABILITY_MASK))
2810 gic_flush_dcache_to_poc(page_address(page), psz);
2811
2812 table[idx] = cpu_to_le64(page_to_phys(page) | GITS_BASER_VALID);
2813
2814 /* Flush Lvl1 entry to PoC if hw doesn't support coherency */
2815 if (!(val & GICR_VPROPBASER_SHAREABILITY_MASK))
2816 gic_flush_dcache_to_poc(table + idx, GITS_LVL1_ENTRY_SIZE);
2817
2818 /* Ensure updated table contents are visible to RD hardware */
2819 dsb(sy);
2820 }
2821
2822 return true;
2823 }
2824
allocate_vpe_l1_table(void)2825 static int allocate_vpe_l1_table(void)
2826 {
2827 void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
2828 u64 val, gpsz, npg, pa;
2829 unsigned int psz = SZ_64K;
2830 unsigned int np, epp, esz;
2831 struct page *page;
2832
2833 if (!gic_rdists->has_rvpeid)
2834 return 0;
2835
2836 /*
2837 * if VPENDBASER.Valid is set, disable any previously programmed
2838 * VPE by setting PendingLast while clearing Valid. This has the
2839 * effect of making sure no doorbell will be generated and we can
2840 * then safely clear VPROPBASER.Valid.
2841 */
2842 if (gicr_read_vpendbaser(vlpi_base + GICR_VPENDBASER) & GICR_VPENDBASER_Valid)
2843 gicr_write_vpendbaser(GICR_VPENDBASER_PendingLast,
2844 vlpi_base + GICR_VPENDBASER);
2845
2846 /*
2847 * If we can inherit the configuration from another RD, let's do
2848 * so. Otherwise, we have to go through the allocation process. We
2849 * assume that all RDs have the exact same requirements, as
2850 * nothing will work otherwise.
2851 */
2852 val = inherit_vpe_l1_table_from_rd(&gic_data_rdist()->vpe_table_mask);
2853 if (val & GICR_VPROPBASER_4_1_VALID)
2854 goto out;
2855
2856 gic_data_rdist()->vpe_table_mask = kzalloc(sizeof(cpumask_t), GFP_ATOMIC);
2857 if (!gic_data_rdist()->vpe_table_mask)
2858 return -ENOMEM;
2859
2860 val = inherit_vpe_l1_table_from_its();
2861 if (val & GICR_VPROPBASER_4_1_VALID)
2862 goto out;
2863
2864 /* First probe the page size */
2865 val = FIELD_PREP(GICR_VPROPBASER_4_1_PAGE_SIZE, GIC_PAGE_SIZE_64K);
2866 gicr_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER);
2867 val = gicr_read_vpropbaser(vlpi_base + GICR_VPROPBASER);
2868 gpsz = FIELD_GET(GICR_VPROPBASER_4_1_PAGE_SIZE, val);
2869 esz = FIELD_GET(GICR_VPROPBASER_4_1_ENTRY_SIZE, val);
2870
2871 switch (gpsz) {
2872 default:
2873 gpsz = GIC_PAGE_SIZE_4K;
2874 fallthrough;
2875 case GIC_PAGE_SIZE_4K:
2876 psz = SZ_4K;
2877 break;
2878 case GIC_PAGE_SIZE_16K:
2879 psz = SZ_16K;
2880 break;
2881 case GIC_PAGE_SIZE_64K:
2882 psz = SZ_64K;
2883 break;
2884 }
2885
2886 /*
2887 * Start populating the register from scratch, including RO fields
2888 * (which we want to print in debug cases...)
2889 */
2890 val = 0;
2891 val |= FIELD_PREP(GICR_VPROPBASER_4_1_PAGE_SIZE, gpsz);
2892 val |= FIELD_PREP(GICR_VPROPBASER_4_1_ENTRY_SIZE, esz);
2893
2894 /* How many entries per GIC page? */
2895 esz++;
2896 epp = psz / (esz * SZ_8);
2897
2898 /*
2899 * If we need more than just a single L1 page, flag the table
2900 * as indirect and compute the number of required L1 pages.
2901 */
2902 if (epp < ITS_MAX_VPEID) {
2903 int nl2;
2904
2905 val |= GICR_VPROPBASER_4_1_INDIRECT;
2906
2907 /* Number of L2 pages required to cover the VPEID space */
2908 nl2 = DIV_ROUND_UP(ITS_MAX_VPEID, epp);
2909
2910 /* Number of L1 pages to point to the L2 pages */
2911 npg = DIV_ROUND_UP(nl2 * SZ_8, psz);
2912 } else {
2913 npg = 1;
2914 }
2915
2916 val |= FIELD_PREP(GICR_VPROPBASER_4_1_SIZE, npg - 1);
2917
2918 /* Right, that's the number of CPU pages we need for L1 */
2919 np = DIV_ROUND_UP(npg * psz, PAGE_SIZE);
2920
2921 pr_debug("np = %d, npg = %lld, psz = %d, epp = %d, esz = %d\n",
2922 np, npg, psz, epp, esz);
2923 page = alloc_pages(GFP_ATOMIC | __GFP_ZERO, get_order(np * PAGE_SIZE));
2924 if (!page)
2925 return -ENOMEM;
2926
2927 gic_data_rdist()->vpe_l1_base = page_address(page);
2928 pa = virt_to_phys(page_address(page));
2929 WARN_ON(!IS_ALIGNED(pa, psz));
2930
2931 val |= FIELD_PREP(GICR_VPROPBASER_4_1_ADDR, pa >> 12);
2932 if (rdists_support_shareable()) {
2933 val |= GICR_VPROPBASER_RaWb;
2934 val |= GICR_VPROPBASER_InnerShareable;
2935 }
2936 val |= GICR_VPROPBASER_4_1_Z;
2937 val |= GICR_VPROPBASER_4_1_VALID;
2938
2939 out:
2940 gicr_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER);
2941 cpumask_set_cpu(smp_processor_id(), gic_data_rdist()->vpe_table_mask);
2942
2943 pr_debug("CPU%d: VPROPBASER = %llx %*pbl\n",
2944 smp_processor_id(), val,
2945 cpumask_pr_args(gic_data_rdist()->vpe_table_mask));
2946
2947 return 0;
2948 }
2949
its_alloc_collections(struct its_node * its)2950 static int its_alloc_collections(struct its_node *its)
2951 {
2952 int i;
2953
2954 its->collections = kcalloc(nr_cpu_ids, sizeof(*its->collections),
2955 GFP_KERNEL);
2956 if (!its->collections)
2957 return -ENOMEM;
2958
2959 for (i = 0; i < nr_cpu_ids; i++)
2960 its->collections[i].target_address = ~0ULL;
2961
2962 return 0;
2963 }
2964
its_allocate_pending_table(gfp_t gfp_flags)2965 static struct page *its_allocate_pending_table(gfp_t gfp_flags)
2966 {
2967 struct page *pend_page;
2968
2969 pend_page = alloc_pages(gfp_flags | __GFP_ZERO,
2970 get_order(LPI_PENDBASE_SZ));
2971 if (!pend_page)
2972 return NULL;
2973
2974 /* Make sure the GIC will observe the zero-ed page */
2975 gic_flush_dcache_to_poc(page_address(pend_page), LPI_PENDBASE_SZ);
2976
2977 return pend_page;
2978 }
2979
its_free_pending_table(struct page * pt)2980 static void its_free_pending_table(struct page *pt)
2981 {
2982 free_pages((unsigned long)page_address(pt), get_order(LPI_PENDBASE_SZ));
2983 }
2984
2985 /*
2986 * Booting with kdump and LPIs enabled is generally fine. Any other
2987 * case is wrong in the absence of firmware/EFI support.
2988 */
enabled_lpis_allowed(void)2989 static bool enabled_lpis_allowed(void)
2990 {
2991 phys_addr_t addr;
2992 u64 val;
2993
2994 /* Check whether the property table is in a reserved region */
2995 val = gicr_read_propbaser(gic_data_rdist_rd_base() + GICR_PROPBASER);
2996 addr = val & GENMASK_ULL(51, 12);
2997
2998 return gic_check_reserved_range(addr, LPI_PROPBASE_SZ);
2999 }
3000
allocate_lpi_tables(void)3001 static int __init allocate_lpi_tables(void)
3002 {
3003 u64 val;
3004 int err, cpu;
3005
3006 /*
3007 * If LPIs are enabled while we run this from the boot CPU,
3008 * flag the RD tables as pre-allocated if the stars do align.
3009 */
3010 val = readl_relaxed(gic_data_rdist_rd_base() + GICR_CTLR);
3011 if ((val & GICR_CTLR_ENABLE_LPIS) && enabled_lpis_allowed()) {
3012 gic_rdists->flags |= (RDIST_FLAGS_RD_TABLES_PREALLOCATED |
3013 RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING);
3014 pr_info("GICv3: Using preallocated redistributor tables\n");
3015 }
3016
3017 err = its_setup_lpi_prop_table();
3018 if (err)
3019 return err;
3020
3021 /*
3022 * We allocate all the pending tables anyway, as we may have a
3023 * mix of RDs that have had LPIs enabled, and some that
3024 * don't. We'll free the unused ones as each CPU comes online.
3025 */
3026 for_each_possible_cpu(cpu) {
3027 struct page *pend_page;
3028
3029 pend_page = its_allocate_pending_table(GFP_NOWAIT);
3030 if (!pend_page) {
3031 pr_err("Failed to allocate PENDBASE for CPU%d\n", cpu);
3032 return -ENOMEM;
3033 }
3034
3035 gic_data_rdist_cpu(cpu)->pend_page = pend_page;
3036 }
3037
3038 return 0;
3039 }
3040
read_vpend_dirty_clear(void __iomem * vlpi_base)3041 static u64 read_vpend_dirty_clear(void __iomem *vlpi_base)
3042 {
3043 u32 count = 1000000; /* 1s! */
3044 bool clean;
3045 u64 val;
3046
3047 do {
3048 val = gicr_read_vpendbaser(vlpi_base + GICR_VPENDBASER);
3049 clean = !(val & GICR_VPENDBASER_Dirty);
3050 if (!clean) {
3051 count--;
3052 cpu_relax();
3053 udelay(1);
3054 }
3055 } while (!clean && count);
3056
3057 if (unlikely(!clean))
3058 pr_err_ratelimited("ITS virtual pending table not cleaning\n");
3059
3060 return val;
3061 }
3062
its_clear_vpend_valid(void __iomem * vlpi_base,u64 clr,u64 set)3063 static u64 its_clear_vpend_valid(void __iomem *vlpi_base, u64 clr, u64 set)
3064 {
3065 u64 val;
3066
3067 /* Make sure we wait until the RD is done with the initial scan */
3068 val = read_vpend_dirty_clear(vlpi_base);
3069 val &= ~GICR_VPENDBASER_Valid;
3070 val &= ~clr;
3071 val |= set;
3072 gicr_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER);
3073
3074 val = read_vpend_dirty_clear(vlpi_base);
3075 if (unlikely(val & GICR_VPENDBASER_Dirty))
3076 val |= GICR_VPENDBASER_PendingLast;
3077
3078 return val;
3079 }
3080
its_cpu_init_lpis(void)3081 static void its_cpu_init_lpis(void)
3082 {
3083 void __iomem *rbase = gic_data_rdist_rd_base();
3084 struct page *pend_page;
3085 phys_addr_t paddr;
3086 u64 val, tmp;
3087
3088 if (gic_data_rdist()->flags & RD_LOCAL_LPI_ENABLED)
3089 return;
3090
3091 val = readl_relaxed(rbase + GICR_CTLR);
3092 if ((gic_rdists->flags & RDIST_FLAGS_RD_TABLES_PREALLOCATED) &&
3093 (val & GICR_CTLR_ENABLE_LPIS)) {
3094 /*
3095 * Check that we get the same property table on all
3096 * RDs. If we don't, this is hopeless.
3097 */
3098 paddr = gicr_read_propbaser(rbase + GICR_PROPBASER);
3099 paddr &= GENMASK_ULL(51, 12);
3100 if (WARN_ON(gic_rdists->prop_table_pa != paddr))
3101 add_taint(TAINT_CRAP, LOCKDEP_STILL_OK);
3102
3103 paddr = gicr_read_pendbaser(rbase + GICR_PENDBASER);
3104 paddr &= GENMASK_ULL(51, 16);
3105
3106 WARN_ON(!gic_check_reserved_range(paddr, LPI_PENDBASE_SZ));
3107 gic_data_rdist()->flags |= RD_LOCAL_PENDTABLE_PREALLOCATED;
3108
3109 goto out;
3110 }
3111
3112 pend_page = gic_data_rdist()->pend_page;
3113 paddr = page_to_phys(pend_page);
3114
3115 /* set PROPBASE */
3116 val = (gic_rdists->prop_table_pa |
3117 GICR_PROPBASER_InnerShareable |
3118 GICR_PROPBASER_RaWaWb |
3119 ((LPI_NRBITS - 1) & GICR_PROPBASER_IDBITS_MASK));
3120
3121 gicr_write_propbaser(val, rbase + GICR_PROPBASER);
3122 tmp = gicr_read_propbaser(rbase + GICR_PROPBASER);
3123
3124 if (!rdists_support_shareable())
3125 tmp &= ~GICR_PROPBASER_SHAREABILITY_MASK;
3126
3127 if ((tmp ^ val) & GICR_PROPBASER_SHAREABILITY_MASK) {
3128 if (!(tmp & GICR_PROPBASER_SHAREABILITY_MASK)) {
3129 /*
3130 * The HW reports non-shareable, we must
3131 * remove the cacheability attributes as
3132 * well.
3133 */
3134 val &= ~(GICR_PROPBASER_SHAREABILITY_MASK |
3135 GICR_PROPBASER_CACHEABILITY_MASK);
3136 val |= GICR_PROPBASER_nC;
3137 gicr_write_propbaser(val, rbase + GICR_PROPBASER);
3138 }
3139 pr_info_once("GIC: using cache flushing for LPI property table\n");
3140 gic_rdists->flags |= RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING;
3141 }
3142
3143 /* set PENDBASE */
3144 val = (page_to_phys(pend_page) |
3145 GICR_PENDBASER_InnerShareable |
3146 GICR_PENDBASER_RaWaWb);
3147
3148 gicr_write_pendbaser(val, rbase + GICR_PENDBASER);
3149 tmp = gicr_read_pendbaser(rbase + GICR_PENDBASER);
3150
3151 if (!rdists_support_shareable())
3152 tmp &= ~GICR_PENDBASER_SHAREABILITY_MASK;
3153
3154 if (!(tmp & GICR_PENDBASER_SHAREABILITY_MASK)) {
3155 /*
3156 * The HW reports non-shareable, we must remove the
3157 * cacheability attributes as well.
3158 */
3159 val &= ~(GICR_PENDBASER_SHAREABILITY_MASK |
3160 GICR_PENDBASER_CACHEABILITY_MASK);
3161 val |= GICR_PENDBASER_nC;
3162 gicr_write_pendbaser(val, rbase + GICR_PENDBASER);
3163 }
3164
3165 /* Enable LPIs */
3166 val = readl_relaxed(rbase + GICR_CTLR);
3167 val |= GICR_CTLR_ENABLE_LPIS;
3168 writel_relaxed(val, rbase + GICR_CTLR);
3169
3170 out:
3171 if (gic_rdists->has_vlpis && !gic_rdists->has_rvpeid) {
3172 void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
3173
3174 /*
3175 * It's possible for CPU to receive VLPIs before it is
3176 * scheduled as a vPE, especially for the first CPU, and the
3177 * VLPI with INTID larger than 2^(IDbits+1) will be considered
3178 * as out of range and dropped by GIC.
3179 * So we initialize IDbits to known value to avoid VLPI drop.
3180 */
3181 val = (LPI_NRBITS - 1) & GICR_VPROPBASER_IDBITS_MASK;
3182 pr_debug("GICv4: CPU%d: Init IDbits to 0x%llx for GICR_VPROPBASER\n",
3183 smp_processor_id(), val);
3184 gicr_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER);
3185
3186 /*
3187 * Also clear Valid bit of GICR_VPENDBASER, in case some
3188 * ancient programming gets left in and has possibility of
3189 * corrupting memory.
3190 */
3191 val = its_clear_vpend_valid(vlpi_base, 0, 0);
3192 }
3193
3194 if (allocate_vpe_l1_table()) {
3195 /*
3196 * If the allocation has failed, we're in massive trouble.
3197 * Disable direct injection, and pray that no VM was
3198 * already running...
3199 */
3200 gic_rdists->has_rvpeid = false;
3201 gic_rdists->has_vlpis = false;
3202 }
3203
3204 /* Make sure the GIC has seen the above */
3205 dsb(sy);
3206 gic_data_rdist()->flags |= RD_LOCAL_LPI_ENABLED;
3207 pr_info("GICv3: CPU%d: using %s LPI pending table @%pa\n",
3208 smp_processor_id(),
3209 gic_data_rdist()->flags & RD_LOCAL_PENDTABLE_PREALLOCATED ?
3210 "reserved" : "allocated",
3211 &paddr);
3212 }
3213
its_cpu_init_collection(struct its_node * its)3214 static void its_cpu_init_collection(struct its_node *its)
3215 {
3216 int cpu = smp_processor_id();
3217 u64 target;
3218
3219 /* avoid cross node collections and its mapping */
3220 if (its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144) {
3221 struct device_node *cpu_node;
3222
3223 cpu_node = of_get_cpu_node(cpu, NULL);
3224 if (its->numa_node != NUMA_NO_NODE &&
3225 its->numa_node != of_node_to_nid(cpu_node))
3226 return;
3227 }
3228
3229 /*
3230 * We now have to bind each collection to its target
3231 * redistributor.
3232 */
3233 if (gic_read_typer(its->base + GITS_TYPER) & GITS_TYPER_PTA) {
3234 /*
3235 * This ITS wants the physical address of the
3236 * redistributor.
3237 */
3238 target = gic_data_rdist()->phys_base;
3239 } else {
3240 /* This ITS wants a linear CPU number. */
3241 target = gic_read_typer(gic_data_rdist_rd_base() + GICR_TYPER);
3242 target = GICR_TYPER_CPU_NUMBER(target) << 16;
3243 }
3244
3245 /* Perform collection mapping */
3246 its->collections[cpu].target_address = target;
3247 its->collections[cpu].col_id = cpu;
3248
3249 its_send_mapc(its, &its->collections[cpu], 1);
3250 its_send_invall(its, &its->collections[cpu]);
3251 }
3252
its_cpu_init_collections(void)3253 static void its_cpu_init_collections(void)
3254 {
3255 struct its_node *its;
3256
3257 raw_spin_lock(&its_lock);
3258
3259 list_for_each_entry(its, &its_nodes, entry)
3260 its_cpu_init_collection(its);
3261
3262 raw_spin_unlock(&its_lock);
3263 }
3264
its_find_device(struct its_node * its,u32 dev_id)3265 static struct its_device *its_find_device(struct its_node *its, u32 dev_id)
3266 {
3267 struct its_device *its_dev = NULL, *tmp;
3268 unsigned long flags;
3269
3270 raw_spin_lock_irqsave(&its->lock, flags);
3271
3272 list_for_each_entry(tmp, &its->its_device_list, entry) {
3273 if (tmp->device_id == dev_id) {
3274 its_dev = tmp;
3275 break;
3276 }
3277 }
3278
3279 raw_spin_unlock_irqrestore(&its->lock, flags);
3280
3281 return its_dev;
3282 }
3283
its_get_baser(struct its_node * its,u32 type)3284 static struct its_baser *its_get_baser(struct its_node *its, u32 type)
3285 {
3286 int i;
3287
3288 for (i = 0; i < GITS_BASER_NR_REGS; i++) {
3289 if (GITS_BASER_TYPE(its->tables[i].val) == type)
3290 return &its->tables[i];
3291 }
3292
3293 return NULL;
3294 }
3295
its_alloc_table_entry(struct its_node * its,struct its_baser * baser,u32 id)3296 static bool its_alloc_table_entry(struct its_node *its,
3297 struct its_baser *baser, u32 id)
3298 {
3299 struct page *page;
3300 u32 esz, idx;
3301 __le64 *table;
3302
3303 /* Don't allow device id that exceeds single, flat table limit */
3304 esz = GITS_BASER_ENTRY_SIZE(baser->val);
3305 if (!(baser->val & GITS_BASER_INDIRECT))
3306 return (id < (PAGE_ORDER_TO_SIZE(baser->order) / esz));
3307
3308 /* Compute 1st level table index & check if that exceeds table limit */
3309 idx = id >> ilog2(baser->psz / esz);
3310 if (idx >= (PAGE_ORDER_TO_SIZE(baser->order) / GITS_LVL1_ENTRY_SIZE))
3311 return false;
3312
3313 table = baser->base;
3314
3315 /* Allocate memory for 2nd level table */
3316 if (!table[idx]) {
3317 page = alloc_pages_node(its->numa_node, GFP_KERNEL | __GFP_ZERO,
3318 get_order(baser->psz));
3319 if (!page)
3320 return false;
3321
3322 /* Flush Lvl2 table to PoC if hw doesn't support coherency */
3323 if (!(baser->val & GITS_BASER_SHAREABILITY_MASK))
3324 gic_flush_dcache_to_poc(page_address(page), baser->psz);
3325
3326 table[idx] = cpu_to_le64(page_to_phys(page) | GITS_BASER_VALID);
3327
3328 /* Flush Lvl1 entry to PoC if hw doesn't support coherency */
3329 if (!(baser->val & GITS_BASER_SHAREABILITY_MASK))
3330 gic_flush_dcache_to_poc(table + idx, GITS_LVL1_ENTRY_SIZE);
3331
3332 /* Ensure updated table contents are visible to ITS hardware */
3333 dsb(sy);
3334 }
3335
3336 return true;
3337 }
3338
its_alloc_device_table(struct its_node * its,u32 dev_id)3339 static bool its_alloc_device_table(struct its_node *its, u32 dev_id)
3340 {
3341 struct its_baser *baser;
3342
3343 baser = its_get_baser(its, GITS_BASER_TYPE_DEVICE);
3344
3345 /* Don't allow device id that exceeds ITS hardware limit */
3346 if (!baser)
3347 return (ilog2(dev_id) < device_ids(its));
3348
3349 return its_alloc_table_entry(its, baser, dev_id);
3350 }
3351
its_alloc_vpe_table(u32 vpe_id)3352 static bool its_alloc_vpe_table(u32 vpe_id)
3353 {
3354 struct its_node *its;
3355 int cpu;
3356
3357 /*
3358 * Make sure the L2 tables are allocated on *all* v4 ITSs. We
3359 * could try and only do it on ITSs corresponding to devices
3360 * that have interrupts targeted at this VPE, but the
3361 * complexity becomes crazy (and you have tons of memory
3362 * anyway, right?).
3363 */
3364 list_for_each_entry(its, &its_nodes, entry) {
3365 struct its_baser *baser;
3366
3367 if (!is_v4(its))
3368 continue;
3369
3370 baser = its_get_baser(its, GITS_BASER_TYPE_VCPU);
3371 if (!baser)
3372 return false;
3373
3374 if (!its_alloc_table_entry(its, baser, vpe_id))
3375 return false;
3376 }
3377
3378 /* Non v4.1? No need to iterate RDs and go back early. */
3379 if (!gic_rdists->has_rvpeid)
3380 return true;
3381
3382 /*
3383 * Make sure the L2 tables are allocated for all copies of
3384 * the L1 table on *all* v4.1 RDs.
3385 */
3386 for_each_possible_cpu(cpu) {
3387 if (!allocate_vpe_l2_table(cpu, vpe_id))
3388 return false;
3389 }
3390
3391 return true;
3392 }
3393
its_create_device(struct its_node * its,u32 dev_id,int nvecs,bool alloc_lpis)3394 static struct its_device *its_create_device(struct its_node *its, u32 dev_id,
3395 int nvecs, bool alloc_lpis)
3396 {
3397 struct its_device *dev;
3398 unsigned long *lpi_map = NULL;
3399 unsigned long flags;
3400 u16 *col_map = NULL;
3401 void *itt;
3402 int lpi_base;
3403 int nr_lpis;
3404 int nr_ites;
3405 int sz;
3406
3407 if (!its_alloc_device_table(its, dev_id))
3408 return NULL;
3409
3410 if (WARN_ON(!is_power_of_2(nvecs)))
3411 nvecs = roundup_pow_of_two(nvecs);
3412
3413 dev = kzalloc(sizeof(*dev), GFP_KERNEL);
3414 /*
3415 * Even if the device wants a single LPI, the ITT must be
3416 * sized as a power of two (and you need at least one bit...).
3417 */
3418 nr_ites = max(2, nvecs);
3419 sz = nr_ites * (FIELD_GET(GITS_TYPER_ITT_ENTRY_SIZE, its->typer) + 1);
3420 sz = max(sz, ITS_ITT_ALIGN) + ITS_ITT_ALIGN - 1;
3421 itt = kzalloc_node(sz, GFP_KERNEL, its->numa_node);
3422 if (alloc_lpis) {
3423 lpi_map = its_lpi_alloc(nvecs, &lpi_base, &nr_lpis);
3424 if (lpi_map)
3425 col_map = kcalloc(nr_lpis, sizeof(*col_map),
3426 GFP_KERNEL);
3427 } else {
3428 col_map = kcalloc(nr_ites, sizeof(*col_map), GFP_KERNEL);
3429 nr_lpis = 0;
3430 lpi_base = 0;
3431 }
3432
3433 if (!dev || !itt || !col_map || (!lpi_map && alloc_lpis)) {
3434 kfree(dev);
3435 kfree(itt);
3436 bitmap_free(lpi_map);
3437 kfree(col_map);
3438 return NULL;
3439 }
3440
3441 gic_flush_dcache_to_poc(itt, sz);
3442
3443 dev->its = its;
3444 dev->itt = itt;
3445 dev->nr_ites = nr_ites;
3446 dev->event_map.lpi_map = lpi_map;
3447 dev->event_map.col_map = col_map;
3448 dev->event_map.lpi_base = lpi_base;
3449 dev->event_map.nr_lpis = nr_lpis;
3450 raw_spin_lock_init(&dev->event_map.vlpi_lock);
3451 dev->device_id = dev_id;
3452 INIT_LIST_HEAD(&dev->entry);
3453
3454 raw_spin_lock_irqsave(&its->lock, flags);
3455 list_add(&dev->entry, &its->its_device_list);
3456 raw_spin_unlock_irqrestore(&its->lock, flags);
3457
3458 /* Map device to its ITT */
3459 its_send_mapd(dev, 1);
3460
3461 return dev;
3462 }
3463
its_free_device(struct its_device * its_dev)3464 static void its_free_device(struct its_device *its_dev)
3465 {
3466 unsigned long flags;
3467
3468 raw_spin_lock_irqsave(&its_dev->its->lock, flags);
3469 list_del(&its_dev->entry);
3470 raw_spin_unlock_irqrestore(&its_dev->its->lock, flags);
3471 kfree(its_dev->event_map.col_map);
3472 kfree(its_dev->itt);
3473 kfree(its_dev);
3474 }
3475
its_alloc_device_irq(struct its_device * dev,int nvecs,irq_hw_number_t * hwirq)3476 static int its_alloc_device_irq(struct its_device *dev, int nvecs, irq_hw_number_t *hwirq)
3477 {
3478 int idx;
3479
3480 /* Find a free LPI region in lpi_map and allocate them. */
3481 idx = bitmap_find_free_region(dev->event_map.lpi_map,
3482 dev->event_map.nr_lpis,
3483 get_count_order(nvecs));
3484 if (idx < 0)
3485 return -ENOSPC;
3486
3487 *hwirq = dev->event_map.lpi_base + idx;
3488
3489 return 0;
3490 }
3491
its_msi_prepare(struct irq_domain * domain,struct device * dev,int nvec,msi_alloc_info_t * info)3492 static int its_msi_prepare(struct irq_domain *domain, struct device *dev,
3493 int nvec, msi_alloc_info_t *info)
3494 {
3495 struct its_node *its;
3496 struct its_device *its_dev;
3497 struct msi_domain_info *msi_info;
3498 u32 dev_id;
3499 int err = 0;
3500
3501 /*
3502 * We ignore "dev" entirely, and rely on the dev_id that has
3503 * been passed via the scratchpad. This limits this domain's
3504 * usefulness to upper layers that definitely know that they
3505 * are built on top of the ITS.
3506 */
3507 dev_id = info->scratchpad[0].ul;
3508
3509 msi_info = msi_get_domain_info(domain);
3510 its = msi_info->data;
3511
3512 if (!gic_rdists->has_direct_lpi &&
3513 vpe_proxy.dev &&
3514 vpe_proxy.dev->its == its &&
3515 dev_id == vpe_proxy.dev->device_id) {
3516 /* Bad luck. Get yourself a better implementation */
3517 WARN_ONCE(1, "DevId %x clashes with GICv4 VPE proxy device\n",
3518 dev_id);
3519 return -EINVAL;
3520 }
3521
3522 mutex_lock(&its->dev_alloc_lock);
3523 its_dev = its_find_device(its, dev_id);
3524 if (its_dev) {
3525 /*
3526 * We already have seen this ID, probably through
3527 * another alias (PCI bridge of some sort). No need to
3528 * create the device.
3529 */
3530 its_dev->shared = true;
3531 pr_debug("Reusing ITT for devID %x\n", dev_id);
3532 goto out;
3533 }
3534
3535 its_dev = its_create_device(its, dev_id, nvec, true);
3536 if (!its_dev) {
3537 err = -ENOMEM;
3538 goto out;
3539 }
3540
3541 if (info->flags & MSI_ALLOC_FLAGS_PROXY_DEVICE)
3542 its_dev->shared = true;
3543
3544 pr_debug("ITT %d entries, %d bits\n", nvec, ilog2(nvec));
3545 out:
3546 mutex_unlock(&its->dev_alloc_lock);
3547 info->scratchpad[0].ptr = its_dev;
3548 return err;
3549 }
3550
3551 static struct msi_domain_ops its_msi_domain_ops = {
3552 .msi_prepare = its_msi_prepare,
3553 };
3554
its_irq_gic_domain_alloc(struct irq_domain * domain,unsigned int virq,irq_hw_number_t hwirq)3555 static int its_irq_gic_domain_alloc(struct irq_domain *domain,
3556 unsigned int virq,
3557 irq_hw_number_t hwirq)
3558 {
3559 struct irq_fwspec fwspec;
3560
3561 if (irq_domain_get_of_node(domain->parent)) {
3562 fwspec.fwnode = domain->parent->fwnode;
3563 fwspec.param_count = 3;
3564 fwspec.param[0] = GIC_IRQ_TYPE_LPI;
3565 fwspec.param[1] = hwirq;
3566 fwspec.param[2] = IRQ_TYPE_EDGE_RISING;
3567 } else if (is_fwnode_irqchip(domain->parent->fwnode)) {
3568 fwspec.fwnode = domain->parent->fwnode;
3569 fwspec.param_count = 2;
3570 fwspec.param[0] = hwirq;
3571 fwspec.param[1] = IRQ_TYPE_EDGE_RISING;
3572 } else {
3573 return -EINVAL;
3574 }
3575
3576 return irq_domain_alloc_irqs_parent(domain, virq, 1, &fwspec);
3577 }
3578
its_irq_domain_alloc(struct irq_domain * domain,unsigned int virq,unsigned int nr_irqs,void * args)3579 static int its_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
3580 unsigned int nr_irqs, void *args)
3581 {
3582 msi_alloc_info_t *info = args;
3583 struct its_device *its_dev = info->scratchpad[0].ptr;
3584 struct its_node *its = its_dev->its;
3585 struct irq_data *irqd;
3586 irq_hw_number_t hwirq;
3587 int err;
3588 int i;
3589
3590 err = its_alloc_device_irq(its_dev, nr_irqs, &hwirq);
3591 if (err)
3592 return err;
3593
3594 err = iommu_dma_prepare_msi(info->desc, its->get_msi_base(its_dev));
3595 if (err)
3596 return err;
3597
3598 for (i = 0; i < nr_irqs; i++) {
3599 err = its_irq_gic_domain_alloc(domain, virq + i, hwirq + i);
3600 if (err)
3601 return err;
3602
3603 irq_domain_set_hwirq_and_chip(domain, virq + i,
3604 hwirq + i, &its_irq_chip, its_dev);
3605 irqd = irq_get_irq_data(virq + i);
3606 irqd_set_single_target(irqd);
3607 irqd_set_affinity_on_activate(irqd);
3608 irqd_set_resend_when_in_progress(irqd);
3609 pr_debug("ID:%d pID:%d vID:%d\n",
3610 (int)(hwirq + i - its_dev->event_map.lpi_base),
3611 (int)(hwirq + i), virq + i);
3612 }
3613
3614 return 0;
3615 }
3616
its_irq_domain_activate(struct irq_domain * domain,struct irq_data * d,bool reserve)3617 static int its_irq_domain_activate(struct irq_domain *domain,
3618 struct irq_data *d, bool reserve)
3619 {
3620 struct its_device *its_dev = irq_data_get_irq_chip_data(d);
3621 u32 event = its_get_event_id(d);
3622 int cpu;
3623
3624 cpu = its_select_cpu(d, cpu_online_mask);
3625 if (cpu < 0 || cpu >= nr_cpu_ids)
3626 return -EINVAL;
3627
3628 its_inc_lpi_count(d, cpu);
3629 its_dev->event_map.col_map[event] = cpu;
3630 irq_data_update_effective_affinity(d, cpumask_of(cpu));
3631
3632 /* Map the GIC IRQ and event to the device */
3633 its_send_mapti(its_dev, d->hwirq, event);
3634 return 0;
3635 }
3636
its_irq_domain_deactivate(struct irq_domain * domain,struct irq_data * d)3637 static void its_irq_domain_deactivate(struct irq_domain *domain,
3638 struct irq_data *d)
3639 {
3640 struct its_device *its_dev = irq_data_get_irq_chip_data(d);
3641 u32 event = its_get_event_id(d);
3642
3643 its_dec_lpi_count(d, its_dev->event_map.col_map[event]);
3644 /* Stop the delivery of interrupts */
3645 its_send_discard(its_dev, event);
3646 }
3647
its_irq_domain_free(struct irq_domain * domain,unsigned int virq,unsigned int nr_irqs)3648 static void its_irq_domain_free(struct irq_domain *domain, unsigned int virq,
3649 unsigned int nr_irqs)
3650 {
3651 struct irq_data *d = irq_domain_get_irq_data(domain, virq);
3652 struct its_device *its_dev = irq_data_get_irq_chip_data(d);
3653 struct its_node *its = its_dev->its;
3654 int i;
3655
3656 bitmap_release_region(its_dev->event_map.lpi_map,
3657 its_get_event_id(irq_domain_get_irq_data(domain, virq)),
3658 get_count_order(nr_irqs));
3659
3660 for (i = 0; i < nr_irqs; i++) {
3661 struct irq_data *data = irq_domain_get_irq_data(domain,
3662 virq + i);
3663 /* Nuke the entry in the domain */
3664 irq_domain_reset_irq_data(data);
3665 }
3666
3667 mutex_lock(&its->dev_alloc_lock);
3668
3669 /*
3670 * If all interrupts have been freed, start mopping the
3671 * floor. This is conditioned on the device not being shared.
3672 */
3673 if (!its_dev->shared &&
3674 bitmap_empty(its_dev->event_map.lpi_map,
3675 its_dev->event_map.nr_lpis)) {
3676 its_lpi_free(its_dev->event_map.lpi_map,
3677 its_dev->event_map.lpi_base,
3678 its_dev->event_map.nr_lpis);
3679
3680 /* Unmap device/itt */
3681 its_send_mapd(its_dev, 0);
3682 its_free_device(its_dev);
3683 }
3684
3685 mutex_unlock(&its->dev_alloc_lock);
3686
3687 irq_domain_free_irqs_parent(domain, virq, nr_irqs);
3688 }
3689
3690 static const struct irq_domain_ops its_domain_ops = {
3691 .alloc = its_irq_domain_alloc,
3692 .free = its_irq_domain_free,
3693 .activate = its_irq_domain_activate,
3694 .deactivate = its_irq_domain_deactivate,
3695 };
3696
3697 /*
3698 * This is insane.
3699 *
3700 * If a GICv4.0 doesn't implement Direct LPIs (which is extremely
3701 * likely), the only way to perform an invalidate is to use a fake
3702 * device to issue an INV command, implying that the LPI has first
3703 * been mapped to some event on that device. Since this is not exactly
3704 * cheap, we try to keep that mapping around as long as possible, and
3705 * only issue an UNMAP if we're short on available slots.
3706 *
3707 * Broken by design(tm).
3708 *
3709 * GICv4.1, on the other hand, mandates that we're able to invalidate
3710 * by writing to a MMIO register. It doesn't implement the whole of
3711 * DirectLPI, but that's good enough. And most of the time, we don't
3712 * even have to invalidate anything, as the redistributor can be told
3713 * whether to generate a doorbell or not (we thus leave it enabled,
3714 * always).
3715 */
its_vpe_db_proxy_unmap_locked(struct its_vpe * vpe)3716 static void its_vpe_db_proxy_unmap_locked(struct its_vpe *vpe)
3717 {
3718 /* GICv4.1 doesn't use a proxy, so nothing to do here */
3719 if (gic_rdists->has_rvpeid)
3720 return;
3721
3722 /* Already unmapped? */
3723 if (vpe->vpe_proxy_event == -1)
3724 return;
3725
3726 its_send_discard(vpe_proxy.dev, vpe->vpe_proxy_event);
3727 vpe_proxy.vpes[vpe->vpe_proxy_event] = NULL;
3728
3729 /*
3730 * We don't track empty slots at all, so let's move the
3731 * next_victim pointer if we can quickly reuse that slot
3732 * instead of nuking an existing entry. Not clear that this is
3733 * always a win though, and this might just generate a ripple
3734 * effect... Let's just hope VPEs don't migrate too often.
3735 */
3736 if (vpe_proxy.vpes[vpe_proxy.next_victim])
3737 vpe_proxy.next_victim = vpe->vpe_proxy_event;
3738
3739 vpe->vpe_proxy_event = -1;
3740 }
3741
its_vpe_db_proxy_unmap(struct its_vpe * vpe)3742 static void its_vpe_db_proxy_unmap(struct its_vpe *vpe)
3743 {
3744 /* GICv4.1 doesn't use a proxy, so nothing to do here */
3745 if (gic_rdists->has_rvpeid)
3746 return;
3747
3748 if (!gic_rdists->has_direct_lpi) {
3749 unsigned long flags;
3750
3751 raw_spin_lock_irqsave(&vpe_proxy.lock, flags);
3752 its_vpe_db_proxy_unmap_locked(vpe);
3753 raw_spin_unlock_irqrestore(&vpe_proxy.lock, flags);
3754 }
3755 }
3756
its_vpe_db_proxy_map_locked(struct its_vpe * vpe)3757 static void its_vpe_db_proxy_map_locked(struct its_vpe *vpe)
3758 {
3759 /* GICv4.1 doesn't use a proxy, so nothing to do here */
3760 if (gic_rdists->has_rvpeid)
3761 return;
3762
3763 /* Already mapped? */
3764 if (vpe->vpe_proxy_event != -1)
3765 return;
3766
3767 /* This slot was already allocated. Kick the other VPE out. */
3768 if (vpe_proxy.vpes[vpe_proxy.next_victim])
3769 its_vpe_db_proxy_unmap_locked(vpe_proxy.vpes[vpe_proxy.next_victim]);
3770
3771 /* Map the new VPE instead */
3772 vpe_proxy.vpes[vpe_proxy.next_victim] = vpe;
3773 vpe->vpe_proxy_event = vpe_proxy.next_victim;
3774 vpe_proxy.next_victim = (vpe_proxy.next_victim + 1) % vpe_proxy.dev->nr_ites;
3775
3776 vpe_proxy.dev->event_map.col_map[vpe->vpe_proxy_event] = vpe->col_idx;
3777 its_send_mapti(vpe_proxy.dev, vpe->vpe_db_lpi, vpe->vpe_proxy_event);
3778 }
3779
its_vpe_db_proxy_move(struct its_vpe * vpe,int from,int to)3780 static void its_vpe_db_proxy_move(struct its_vpe *vpe, int from, int to)
3781 {
3782 unsigned long flags;
3783 struct its_collection *target_col;
3784
3785 /* GICv4.1 doesn't use a proxy, so nothing to do here */
3786 if (gic_rdists->has_rvpeid)
3787 return;
3788
3789 if (gic_rdists->has_direct_lpi) {
3790 void __iomem *rdbase;
3791
3792 rdbase = per_cpu_ptr(gic_rdists->rdist, from)->rd_base;
3793 gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_CLRLPIR);
3794 wait_for_syncr(rdbase);
3795
3796 return;
3797 }
3798
3799 raw_spin_lock_irqsave(&vpe_proxy.lock, flags);
3800
3801 its_vpe_db_proxy_map_locked(vpe);
3802
3803 target_col = &vpe_proxy.dev->its->collections[to];
3804 its_send_movi(vpe_proxy.dev, target_col, vpe->vpe_proxy_event);
3805 vpe_proxy.dev->event_map.col_map[vpe->vpe_proxy_event] = to;
3806
3807 raw_spin_unlock_irqrestore(&vpe_proxy.lock, flags);
3808 }
3809
its_vpe_set_affinity(struct irq_data * d,const struct cpumask * mask_val,bool force)3810 static int its_vpe_set_affinity(struct irq_data *d,
3811 const struct cpumask *mask_val,
3812 bool force)
3813 {
3814 struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
3815 struct cpumask common, *table_mask;
3816 unsigned long flags;
3817 int from, cpu;
3818
3819 /*
3820 * Check if we're racing against a VPE being destroyed, for
3821 * which we don't want to allow a VMOVP.
3822 */
3823 if (!atomic_read(&vpe->vmapp_count))
3824 return -EINVAL;
3825
3826 /*
3827 * Changing affinity is mega expensive, so let's be as lazy as
3828 * we can and only do it if we really have to. Also, if mapped
3829 * into the proxy device, we need to move the doorbell
3830 * interrupt to its new location.
3831 *
3832 * Another thing is that changing the affinity of a vPE affects
3833 * *other interrupts* such as all the vLPIs that are routed to
3834 * this vPE. This means that the irq_desc lock is not enough to
3835 * protect us, and that we must ensure nobody samples vpe->col_idx
3836 * during the update, hence the lock below which must also be
3837 * taken on any vLPI handling path that evaluates vpe->col_idx.
3838 */
3839 from = vpe_to_cpuid_lock(vpe, &flags);
3840 table_mask = gic_data_rdist_cpu(from)->vpe_table_mask;
3841
3842 /*
3843 * If we are offered another CPU in the same GICv4.1 ITS
3844 * affinity, pick this one. Otherwise, any CPU will do.
3845 */
3846 if (table_mask && cpumask_and(&common, mask_val, table_mask))
3847 cpu = cpumask_test_cpu(from, &common) ? from : cpumask_first(&common);
3848 else
3849 cpu = cpumask_first(mask_val);
3850
3851 if (from == cpu)
3852 goto out;
3853
3854 vpe->col_idx = cpu;
3855
3856 its_send_vmovp(vpe);
3857 its_vpe_db_proxy_move(vpe, from, cpu);
3858
3859 out:
3860 irq_data_update_effective_affinity(d, cpumask_of(cpu));
3861 vpe_to_cpuid_unlock(vpe, flags);
3862
3863 return IRQ_SET_MASK_OK_DONE;
3864 }
3865
its_wait_vpt_parse_complete(void)3866 static void its_wait_vpt_parse_complete(void)
3867 {
3868 void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
3869 u64 val;
3870
3871 if (!gic_rdists->has_vpend_valid_dirty)
3872 return;
3873
3874 WARN_ON_ONCE(readq_relaxed_poll_timeout_atomic(vlpi_base + GICR_VPENDBASER,
3875 val,
3876 !(val & GICR_VPENDBASER_Dirty),
3877 1, 500));
3878 }
3879
its_vpe_schedule(struct its_vpe * vpe)3880 static void its_vpe_schedule(struct its_vpe *vpe)
3881 {
3882 void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
3883 u64 val;
3884
3885 /* Schedule the VPE */
3886 val = virt_to_phys(page_address(vpe->its_vm->vprop_page)) &
3887 GENMASK_ULL(51, 12);
3888 val |= (LPI_NRBITS - 1) & GICR_VPROPBASER_IDBITS_MASK;
3889 if (rdists_support_shareable()) {
3890 val |= GICR_VPROPBASER_RaWb;
3891 val |= GICR_VPROPBASER_InnerShareable;
3892 }
3893 gicr_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER);
3894
3895 val = virt_to_phys(page_address(vpe->vpt_page)) &
3896 GENMASK_ULL(51, 16);
3897 if (rdists_support_shareable()) {
3898 val |= GICR_VPENDBASER_RaWaWb;
3899 val |= GICR_VPENDBASER_InnerShareable;
3900 }
3901 /*
3902 * There is no good way of finding out if the pending table is
3903 * empty as we can race against the doorbell interrupt very
3904 * easily. So in the end, vpe->pending_last is only an
3905 * indication that the vcpu has something pending, not one
3906 * that the pending table is empty. A good implementation
3907 * would be able to read its coarse map pretty quickly anyway,
3908 * making this a tolerable issue.
3909 */
3910 val |= GICR_VPENDBASER_PendingLast;
3911 val |= vpe->idai ? GICR_VPENDBASER_IDAI : 0;
3912 val |= GICR_VPENDBASER_Valid;
3913 gicr_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER);
3914 }
3915
its_vpe_deschedule(struct its_vpe * vpe)3916 static void its_vpe_deschedule(struct its_vpe *vpe)
3917 {
3918 void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
3919 u64 val;
3920
3921 val = its_clear_vpend_valid(vlpi_base, 0, 0);
3922
3923 vpe->idai = !!(val & GICR_VPENDBASER_IDAI);
3924 vpe->pending_last = !!(val & GICR_VPENDBASER_PendingLast);
3925 }
3926
its_vpe_invall(struct its_vpe * vpe)3927 static void its_vpe_invall(struct its_vpe *vpe)
3928 {
3929 struct its_node *its;
3930
3931 list_for_each_entry(its, &its_nodes, entry) {
3932 if (!is_v4(its))
3933 continue;
3934
3935 if (its_list_map && !vpe->its_vm->vlpi_count[its->list_nr])
3936 continue;
3937
3938 /*
3939 * Sending a VINVALL to a single ITS is enough, as all
3940 * we need is to reach the redistributors.
3941 */
3942 its_send_vinvall(its, vpe);
3943 return;
3944 }
3945 }
3946
its_vpe_set_vcpu_affinity(struct irq_data * d,void * vcpu_info)3947 static int its_vpe_set_vcpu_affinity(struct irq_data *d, void *vcpu_info)
3948 {
3949 struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
3950 struct its_cmd_info *info = vcpu_info;
3951
3952 switch (info->cmd_type) {
3953 case SCHEDULE_VPE:
3954 its_vpe_schedule(vpe);
3955 return 0;
3956
3957 case DESCHEDULE_VPE:
3958 its_vpe_deschedule(vpe);
3959 return 0;
3960
3961 case COMMIT_VPE:
3962 its_wait_vpt_parse_complete();
3963 return 0;
3964
3965 case INVALL_VPE:
3966 its_vpe_invall(vpe);
3967 return 0;
3968
3969 default:
3970 return -EINVAL;
3971 }
3972 }
3973
its_vpe_send_cmd(struct its_vpe * vpe,void (* cmd)(struct its_device *,u32))3974 static void its_vpe_send_cmd(struct its_vpe *vpe,
3975 void (*cmd)(struct its_device *, u32))
3976 {
3977 unsigned long flags;
3978
3979 raw_spin_lock_irqsave(&vpe_proxy.lock, flags);
3980
3981 its_vpe_db_proxy_map_locked(vpe);
3982 cmd(vpe_proxy.dev, vpe->vpe_proxy_event);
3983
3984 raw_spin_unlock_irqrestore(&vpe_proxy.lock, flags);
3985 }
3986
its_vpe_send_inv(struct irq_data * d)3987 static void its_vpe_send_inv(struct irq_data *d)
3988 {
3989 struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
3990
3991 if (gic_rdists->has_direct_lpi)
3992 __direct_lpi_inv(d, d->parent_data->hwirq);
3993 else
3994 its_vpe_send_cmd(vpe, its_send_inv);
3995 }
3996
its_vpe_mask_irq(struct irq_data * d)3997 static void its_vpe_mask_irq(struct irq_data *d)
3998 {
3999 /*
4000 * We need to unmask the LPI, which is described by the parent
4001 * irq_data. Instead of calling into the parent (which won't
4002 * exactly do the right thing, let's simply use the
4003 * parent_data pointer. Yes, I'm naughty.
4004 */
4005 lpi_write_config(d->parent_data, LPI_PROP_ENABLED, 0);
4006 its_vpe_send_inv(d);
4007 }
4008
its_vpe_unmask_irq(struct irq_data * d)4009 static void its_vpe_unmask_irq(struct irq_data *d)
4010 {
4011 /* Same hack as above... */
4012 lpi_write_config(d->parent_data, 0, LPI_PROP_ENABLED);
4013 its_vpe_send_inv(d);
4014 }
4015
its_vpe_set_irqchip_state(struct irq_data * d,enum irqchip_irq_state which,bool state)4016 static int its_vpe_set_irqchip_state(struct irq_data *d,
4017 enum irqchip_irq_state which,
4018 bool state)
4019 {
4020 struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4021
4022 if (which != IRQCHIP_STATE_PENDING)
4023 return -EINVAL;
4024
4025 if (gic_rdists->has_direct_lpi) {
4026 void __iomem *rdbase;
4027
4028 rdbase = per_cpu_ptr(gic_rdists->rdist, vpe->col_idx)->rd_base;
4029 if (state) {
4030 gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_SETLPIR);
4031 } else {
4032 gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_CLRLPIR);
4033 wait_for_syncr(rdbase);
4034 }
4035 } else {
4036 if (state)
4037 its_vpe_send_cmd(vpe, its_send_int);
4038 else
4039 its_vpe_send_cmd(vpe, its_send_clear);
4040 }
4041
4042 return 0;
4043 }
4044
its_vpe_retrigger(struct irq_data * d)4045 static int its_vpe_retrigger(struct irq_data *d)
4046 {
4047 return !its_vpe_set_irqchip_state(d, IRQCHIP_STATE_PENDING, true);
4048 }
4049
4050 static struct irq_chip its_vpe_irq_chip = {
4051 .name = "GICv4-vpe",
4052 .irq_mask = its_vpe_mask_irq,
4053 .irq_unmask = its_vpe_unmask_irq,
4054 .irq_eoi = irq_chip_eoi_parent,
4055 .irq_set_affinity = its_vpe_set_affinity,
4056 .irq_retrigger = its_vpe_retrigger,
4057 .irq_set_irqchip_state = its_vpe_set_irqchip_state,
4058 .irq_set_vcpu_affinity = its_vpe_set_vcpu_affinity,
4059 };
4060
find_4_1_its(void)4061 static struct its_node *find_4_1_its(void)
4062 {
4063 static struct its_node *its = NULL;
4064
4065 if (!its) {
4066 list_for_each_entry(its, &its_nodes, entry) {
4067 if (is_v4_1(its))
4068 return its;
4069 }
4070
4071 /* Oops? */
4072 its = NULL;
4073 }
4074
4075 return its;
4076 }
4077
its_vpe_4_1_send_inv(struct irq_data * d)4078 static void its_vpe_4_1_send_inv(struct irq_data *d)
4079 {
4080 struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4081 struct its_node *its;
4082
4083 /*
4084 * GICv4.1 wants doorbells to be invalidated using the
4085 * INVDB command in order to be broadcast to all RDs. Send
4086 * it to the first valid ITS, and let the HW do its magic.
4087 */
4088 its = find_4_1_its();
4089 if (its)
4090 its_send_invdb(its, vpe);
4091 }
4092
its_vpe_4_1_mask_irq(struct irq_data * d)4093 static void its_vpe_4_1_mask_irq(struct irq_data *d)
4094 {
4095 lpi_write_config(d->parent_data, LPI_PROP_ENABLED, 0);
4096 its_vpe_4_1_send_inv(d);
4097 }
4098
its_vpe_4_1_unmask_irq(struct irq_data * d)4099 static void its_vpe_4_1_unmask_irq(struct irq_data *d)
4100 {
4101 lpi_write_config(d->parent_data, 0, LPI_PROP_ENABLED);
4102 its_vpe_4_1_send_inv(d);
4103 }
4104
its_vpe_4_1_schedule(struct its_vpe * vpe,struct its_cmd_info * info)4105 static void its_vpe_4_1_schedule(struct its_vpe *vpe,
4106 struct its_cmd_info *info)
4107 {
4108 void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
4109 u64 val = 0;
4110
4111 /* Schedule the VPE */
4112 val |= GICR_VPENDBASER_Valid;
4113 val |= info->g0en ? GICR_VPENDBASER_4_1_VGRP0EN : 0;
4114 val |= info->g1en ? GICR_VPENDBASER_4_1_VGRP1EN : 0;
4115 val |= FIELD_PREP(GICR_VPENDBASER_4_1_VPEID, vpe->vpe_id);
4116
4117 gicr_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER);
4118 }
4119
its_vpe_4_1_deschedule(struct its_vpe * vpe,struct its_cmd_info * info)4120 static void its_vpe_4_1_deschedule(struct its_vpe *vpe,
4121 struct its_cmd_info *info)
4122 {
4123 void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
4124 u64 val;
4125
4126 if (info->req_db) {
4127 unsigned long flags;
4128
4129 /*
4130 * vPE is going to block: make the vPE non-resident with
4131 * PendingLast clear and DB set. The GIC guarantees that if
4132 * we read-back PendingLast clear, then a doorbell will be
4133 * delivered when an interrupt comes.
4134 *
4135 * Note the locking to deal with the concurrent update of
4136 * pending_last from the doorbell interrupt handler that can
4137 * run concurrently.
4138 */
4139 raw_spin_lock_irqsave(&vpe->vpe_lock, flags);
4140 val = its_clear_vpend_valid(vlpi_base,
4141 GICR_VPENDBASER_PendingLast,
4142 GICR_VPENDBASER_4_1_DB);
4143 vpe->pending_last = !!(val & GICR_VPENDBASER_PendingLast);
4144 raw_spin_unlock_irqrestore(&vpe->vpe_lock, flags);
4145 } else {
4146 /*
4147 * We're not blocking, so just make the vPE non-resident
4148 * with PendingLast set, indicating that we'll be back.
4149 */
4150 val = its_clear_vpend_valid(vlpi_base,
4151 0,
4152 GICR_VPENDBASER_PendingLast);
4153 vpe->pending_last = true;
4154 }
4155 }
4156
its_vpe_4_1_invall(struct its_vpe * vpe)4157 static void its_vpe_4_1_invall(struct its_vpe *vpe)
4158 {
4159 void __iomem *rdbase;
4160 unsigned long flags;
4161 u64 val;
4162 int cpu;
4163
4164 val = GICR_INVALLR_V;
4165 val |= FIELD_PREP(GICR_INVALLR_VPEID, vpe->vpe_id);
4166
4167 /* Target the redistributor this vPE is currently known on */
4168 cpu = vpe_to_cpuid_lock(vpe, &flags);
4169 raw_spin_lock(&gic_data_rdist_cpu(cpu)->rd_lock);
4170 rdbase = per_cpu_ptr(gic_rdists->rdist, cpu)->rd_base;
4171 gic_write_lpir(val, rdbase + GICR_INVALLR);
4172
4173 wait_for_syncr(rdbase);
4174 raw_spin_unlock(&gic_data_rdist_cpu(cpu)->rd_lock);
4175 vpe_to_cpuid_unlock(vpe, flags);
4176 }
4177
its_vpe_4_1_set_vcpu_affinity(struct irq_data * d,void * vcpu_info)4178 static int its_vpe_4_1_set_vcpu_affinity(struct irq_data *d, void *vcpu_info)
4179 {
4180 struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4181 struct its_cmd_info *info = vcpu_info;
4182
4183 switch (info->cmd_type) {
4184 case SCHEDULE_VPE:
4185 its_vpe_4_1_schedule(vpe, info);
4186 return 0;
4187
4188 case DESCHEDULE_VPE:
4189 its_vpe_4_1_deschedule(vpe, info);
4190 return 0;
4191
4192 case COMMIT_VPE:
4193 its_wait_vpt_parse_complete();
4194 return 0;
4195
4196 case INVALL_VPE:
4197 its_vpe_4_1_invall(vpe);
4198 return 0;
4199
4200 default:
4201 return -EINVAL;
4202 }
4203 }
4204
4205 static struct irq_chip its_vpe_4_1_irq_chip = {
4206 .name = "GICv4.1-vpe",
4207 .irq_mask = its_vpe_4_1_mask_irq,
4208 .irq_unmask = its_vpe_4_1_unmask_irq,
4209 .irq_eoi = irq_chip_eoi_parent,
4210 .irq_set_affinity = its_vpe_set_affinity,
4211 .irq_set_vcpu_affinity = its_vpe_4_1_set_vcpu_affinity,
4212 };
4213
its_configure_sgi(struct irq_data * d,bool clear)4214 static void its_configure_sgi(struct irq_data *d, bool clear)
4215 {
4216 struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4217 struct its_cmd_desc desc;
4218
4219 desc.its_vsgi_cmd.vpe = vpe;
4220 desc.its_vsgi_cmd.sgi = d->hwirq;
4221 desc.its_vsgi_cmd.priority = vpe->sgi_config[d->hwirq].priority;
4222 desc.its_vsgi_cmd.enable = vpe->sgi_config[d->hwirq].enabled;
4223 desc.its_vsgi_cmd.group = vpe->sgi_config[d->hwirq].group;
4224 desc.its_vsgi_cmd.clear = clear;
4225
4226 /*
4227 * GICv4.1 allows us to send VSGI commands to any ITS as long as the
4228 * destination VPE is mapped there. Since we map them eagerly at
4229 * activation time, we're pretty sure the first GICv4.1 ITS will do.
4230 */
4231 its_send_single_vcommand(find_4_1_its(), its_build_vsgi_cmd, &desc);
4232 }
4233
its_sgi_mask_irq(struct irq_data * d)4234 static void its_sgi_mask_irq(struct irq_data *d)
4235 {
4236 struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4237
4238 vpe->sgi_config[d->hwirq].enabled = false;
4239 its_configure_sgi(d, false);
4240 }
4241
its_sgi_unmask_irq(struct irq_data * d)4242 static void its_sgi_unmask_irq(struct irq_data *d)
4243 {
4244 struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4245
4246 vpe->sgi_config[d->hwirq].enabled = true;
4247 its_configure_sgi(d, false);
4248 }
4249
its_sgi_set_affinity(struct irq_data * d,const struct cpumask * mask_val,bool force)4250 static int its_sgi_set_affinity(struct irq_data *d,
4251 const struct cpumask *mask_val,
4252 bool force)
4253 {
4254 /*
4255 * There is no notion of affinity for virtual SGIs, at least
4256 * not on the host (since they can only be targeting a vPE).
4257 * Tell the kernel we've done whatever it asked for.
4258 */
4259 irq_data_update_effective_affinity(d, mask_val);
4260 return IRQ_SET_MASK_OK;
4261 }
4262
its_sgi_set_irqchip_state(struct irq_data * d,enum irqchip_irq_state which,bool state)4263 static int its_sgi_set_irqchip_state(struct irq_data *d,
4264 enum irqchip_irq_state which,
4265 bool state)
4266 {
4267 if (which != IRQCHIP_STATE_PENDING)
4268 return -EINVAL;
4269
4270 if (state) {
4271 struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4272 struct its_node *its = find_4_1_its();
4273 u64 val;
4274
4275 val = FIELD_PREP(GITS_SGIR_VPEID, vpe->vpe_id);
4276 val |= FIELD_PREP(GITS_SGIR_VINTID, d->hwirq);
4277 writeq_relaxed(val, its->sgir_base + GITS_SGIR - SZ_128K);
4278 } else {
4279 its_configure_sgi(d, true);
4280 }
4281
4282 return 0;
4283 }
4284
its_sgi_get_irqchip_state(struct irq_data * d,enum irqchip_irq_state which,bool * val)4285 static int its_sgi_get_irqchip_state(struct irq_data *d,
4286 enum irqchip_irq_state which, bool *val)
4287 {
4288 struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4289 void __iomem *base;
4290 unsigned long flags;
4291 u32 count = 1000000; /* 1s! */
4292 u32 status;
4293 int cpu;
4294
4295 if (which != IRQCHIP_STATE_PENDING)
4296 return -EINVAL;
4297
4298 /*
4299 * Locking galore! We can race against two different events:
4300 *
4301 * - Concurrent vPE affinity change: we must make sure it cannot
4302 * happen, or we'll talk to the wrong redistributor. This is
4303 * identical to what happens with vLPIs.
4304 *
4305 * - Concurrent VSGIPENDR access: As it involves accessing two
4306 * MMIO registers, this must be made atomic one way or another.
4307 */
4308 cpu = vpe_to_cpuid_lock(vpe, &flags);
4309 raw_spin_lock(&gic_data_rdist_cpu(cpu)->rd_lock);
4310 base = gic_data_rdist_cpu(cpu)->rd_base + SZ_128K;
4311 writel_relaxed(vpe->vpe_id, base + GICR_VSGIR);
4312 do {
4313 status = readl_relaxed(base + GICR_VSGIPENDR);
4314 if (!(status & GICR_VSGIPENDR_BUSY))
4315 goto out;
4316
4317 count--;
4318 if (!count) {
4319 pr_err_ratelimited("Unable to get SGI status\n");
4320 goto out;
4321 }
4322 cpu_relax();
4323 udelay(1);
4324 } while (count);
4325
4326 out:
4327 raw_spin_unlock(&gic_data_rdist_cpu(cpu)->rd_lock);
4328 vpe_to_cpuid_unlock(vpe, flags);
4329
4330 if (!count)
4331 return -ENXIO;
4332
4333 *val = !!(status & (1 << d->hwirq));
4334
4335 return 0;
4336 }
4337
its_sgi_set_vcpu_affinity(struct irq_data * d,void * vcpu_info)4338 static int its_sgi_set_vcpu_affinity(struct irq_data *d, void *vcpu_info)
4339 {
4340 struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4341 struct its_cmd_info *info = vcpu_info;
4342
4343 switch (info->cmd_type) {
4344 case PROP_UPDATE_VSGI:
4345 vpe->sgi_config[d->hwirq].priority = info->priority;
4346 vpe->sgi_config[d->hwirq].group = info->group;
4347 its_configure_sgi(d, false);
4348 return 0;
4349
4350 default:
4351 return -EINVAL;
4352 }
4353 }
4354
4355 static struct irq_chip its_sgi_irq_chip = {
4356 .name = "GICv4.1-sgi",
4357 .irq_mask = its_sgi_mask_irq,
4358 .irq_unmask = its_sgi_unmask_irq,
4359 .irq_set_affinity = its_sgi_set_affinity,
4360 .irq_set_irqchip_state = its_sgi_set_irqchip_state,
4361 .irq_get_irqchip_state = its_sgi_get_irqchip_state,
4362 .irq_set_vcpu_affinity = its_sgi_set_vcpu_affinity,
4363 };
4364
its_sgi_irq_domain_alloc(struct irq_domain * domain,unsigned int virq,unsigned int nr_irqs,void * args)4365 static int its_sgi_irq_domain_alloc(struct irq_domain *domain,
4366 unsigned int virq, unsigned int nr_irqs,
4367 void *args)
4368 {
4369 struct its_vpe *vpe = args;
4370 int i;
4371
4372 /* Yes, we do want 16 SGIs */
4373 WARN_ON(nr_irqs != 16);
4374
4375 for (i = 0; i < 16; i++) {
4376 vpe->sgi_config[i].priority = 0;
4377 vpe->sgi_config[i].enabled = false;
4378 vpe->sgi_config[i].group = false;
4379
4380 irq_domain_set_hwirq_and_chip(domain, virq + i, i,
4381 &its_sgi_irq_chip, vpe);
4382 irq_set_status_flags(virq + i, IRQ_DISABLE_UNLAZY);
4383 }
4384
4385 return 0;
4386 }
4387
its_sgi_irq_domain_free(struct irq_domain * domain,unsigned int virq,unsigned int nr_irqs)4388 static void its_sgi_irq_domain_free(struct irq_domain *domain,
4389 unsigned int virq,
4390 unsigned int nr_irqs)
4391 {
4392 /* Nothing to do */
4393 }
4394
its_sgi_irq_domain_activate(struct irq_domain * domain,struct irq_data * d,bool reserve)4395 static int its_sgi_irq_domain_activate(struct irq_domain *domain,
4396 struct irq_data *d, bool reserve)
4397 {
4398 /* Write out the initial SGI configuration */
4399 its_configure_sgi(d, false);
4400 return 0;
4401 }
4402
its_sgi_irq_domain_deactivate(struct irq_domain * domain,struct irq_data * d)4403 static void its_sgi_irq_domain_deactivate(struct irq_domain *domain,
4404 struct irq_data *d)
4405 {
4406 struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4407
4408 /*
4409 * The VSGI command is awkward:
4410 *
4411 * - To change the configuration, CLEAR must be set to false,
4412 * leaving the pending bit unchanged.
4413 * - To clear the pending bit, CLEAR must be set to true, leaving
4414 * the configuration unchanged.
4415 *
4416 * You just can't do both at once, hence the two commands below.
4417 */
4418 vpe->sgi_config[d->hwirq].enabled = false;
4419 its_configure_sgi(d, false);
4420 its_configure_sgi(d, true);
4421 }
4422
4423 static const struct irq_domain_ops its_sgi_domain_ops = {
4424 .alloc = its_sgi_irq_domain_alloc,
4425 .free = its_sgi_irq_domain_free,
4426 .activate = its_sgi_irq_domain_activate,
4427 .deactivate = its_sgi_irq_domain_deactivate,
4428 };
4429
its_vpe_id_alloc(void)4430 static int its_vpe_id_alloc(void)
4431 {
4432 return ida_simple_get(&its_vpeid_ida, 0, ITS_MAX_VPEID, GFP_KERNEL);
4433 }
4434
its_vpe_id_free(u16 id)4435 static void its_vpe_id_free(u16 id)
4436 {
4437 ida_simple_remove(&its_vpeid_ida, id);
4438 }
4439
its_vpe_init(struct its_vpe * vpe)4440 static int its_vpe_init(struct its_vpe *vpe)
4441 {
4442 struct page *vpt_page;
4443 int vpe_id;
4444
4445 /* Allocate vpe_id */
4446 vpe_id = its_vpe_id_alloc();
4447 if (vpe_id < 0)
4448 return vpe_id;
4449
4450 /* Allocate VPT */
4451 vpt_page = its_allocate_pending_table(GFP_KERNEL);
4452 if (!vpt_page) {
4453 its_vpe_id_free(vpe_id);
4454 return -ENOMEM;
4455 }
4456
4457 if (!its_alloc_vpe_table(vpe_id)) {
4458 its_vpe_id_free(vpe_id);
4459 its_free_pending_table(vpt_page);
4460 return -ENOMEM;
4461 }
4462
4463 raw_spin_lock_init(&vpe->vpe_lock);
4464 vpe->vpe_id = vpe_id;
4465 vpe->vpt_page = vpt_page;
4466 atomic_set(&vpe->vmapp_count, 0);
4467 if (!gic_rdists->has_rvpeid)
4468 vpe->vpe_proxy_event = -1;
4469
4470 return 0;
4471 }
4472
its_vpe_teardown(struct its_vpe * vpe)4473 static void its_vpe_teardown(struct its_vpe *vpe)
4474 {
4475 its_vpe_db_proxy_unmap(vpe);
4476 its_vpe_id_free(vpe->vpe_id);
4477 its_free_pending_table(vpe->vpt_page);
4478 }
4479
its_vpe_irq_domain_free(struct irq_domain * domain,unsigned int virq,unsigned int nr_irqs)4480 static void its_vpe_irq_domain_free(struct irq_domain *domain,
4481 unsigned int virq,
4482 unsigned int nr_irqs)
4483 {
4484 struct its_vm *vm = domain->host_data;
4485 int i;
4486
4487 irq_domain_free_irqs_parent(domain, virq, nr_irqs);
4488
4489 for (i = 0; i < nr_irqs; i++) {
4490 struct irq_data *data = irq_domain_get_irq_data(domain,
4491 virq + i);
4492 struct its_vpe *vpe = irq_data_get_irq_chip_data(data);
4493
4494 BUG_ON(vm != vpe->its_vm);
4495
4496 clear_bit(data->hwirq, vm->db_bitmap);
4497 its_vpe_teardown(vpe);
4498 irq_domain_reset_irq_data(data);
4499 }
4500
4501 if (bitmap_empty(vm->db_bitmap, vm->nr_db_lpis)) {
4502 its_lpi_free(vm->db_bitmap, vm->db_lpi_base, vm->nr_db_lpis);
4503 its_free_prop_table(vm->vprop_page);
4504 }
4505 }
4506
its_vpe_irq_domain_alloc(struct irq_domain * domain,unsigned int virq,unsigned int nr_irqs,void * args)4507 static int its_vpe_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
4508 unsigned int nr_irqs, void *args)
4509 {
4510 struct irq_chip *irqchip = &its_vpe_irq_chip;
4511 struct its_vm *vm = args;
4512 unsigned long *bitmap;
4513 struct page *vprop_page;
4514 int base, nr_ids, i, err = 0;
4515
4516 bitmap = its_lpi_alloc(roundup_pow_of_two(nr_irqs), &base, &nr_ids);
4517 if (!bitmap)
4518 return -ENOMEM;
4519
4520 if (nr_ids < nr_irqs) {
4521 its_lpi_free(bitmap, base, nr_ids);
4522 return -ENOMEM;
4523 }
4524
4525 vprop_page = its_allocate_prop_table(GFP_KERNEL);
4526 if (!vprop_page) {
4527 its_lpi_free(bitmap, base, nr_ids);
4528 return -ENOMEM;
4529 }
4530
4531 vm->db_bitmap = bitmap;
4532 vm->db_lpi_base = base;
4533 vm->nr_db_lpis = nr_ids;
4534 vm->vprop_page = vprop_page;
4535
4536 if (gic_rdists->has_rvpeid)
4537 irqchip = &its_vpe_4_1_irq_chip;
4538
4539 for (i = 0; i < nr_irqs; i++) {
4540 vm->vpes[i]->vpe_db_lpi = base + i;
4541 err = its_vpe_init(vm->vpes[i]);
4542 if (err)
4543 break;
4544 err = its_irq_gic_domain_alloc(domain, virq + i,
4545 vm->vpes[i]->vpe_db_lpi);
4546 if (err)
4547 break;
4548 irq_domain_set_hwirq_and_chip(domain, virq + i, i,
4549 irqchip, vm->vpes[i]);
4550 set_bit(i, bitmap);
4551 irqd_set_resend_when_in_progress(irq_get_irq_data(virq + i));
4552 }
4553
4554 if (err)
4555 its_vpe_irq_domain_free(domain, virq, i);
4556
4557 return err;
4558 }
4559
its_vpe_irq_domain_activate(struct irq_domain * domain,struct irq_data * d,bool reserve)4560 static int its_vpe_irq_domain_activate(struct irq_domain *domain,
4561 struct irq_data *d, bool reserve)
4562 {
4563 struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4564 struct its_node *its;
4565
4566 /*
4567 * If we use the list map, we issue VMAPP on demand... Unless
4568 * we're on a GICv4.1 and we eagerly map the VPE on all ITSs
4569 * so that VSGIs can work.
4570 */
4571 if (!gic_requires_eager_mapping())
4572 return 0;
4573
4574 /* Map the VPE to the first possible CPU */
4575 vpe->col_idx = cpumask_first(cpu_online_mask);
4576
4577 list_for_each_entry(its, &its_nodes, entry) {
4578 if (!is_v4(its))
4579 continue;
4580
4581 its_send_vmapp(its, vpe, true);
4582 its_send_vinvall(its, vpe);
4583 }
4584
4585 irq_data_update_effective_affinity(d, cpumask_of(vpe->col_idx));
4586
4587 return 0;
4588 }
4589
its_vpe_irq_domain_deactivate(struct irq_domain * domain,struct irq_data * d)4590 static void its_vpe_irq_domain_deactivate(struct irq_domain *domain,
4591 struct irq_data *d)
4592 {
4593 struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4594 struct its_node *its;
4595
4596 /*
4597 * If we use the list map on GICv4.0, we unmap the VPE once no
4598 * VLPIs are associated with the VM.
4599 */
4600 if (!gic_requires_eager_mapping())
4601 return;
4602
4603 list_for_each_entry(its, &its_nodes, entry) {
4604 if (!is_v4(its))
4605 continue;
4606
4607 its_send_vmapp(its, vpe, false);
4608 }
4609
4610 /*
4611 * There may be a direct read to the VPT after unmapping the
4612 * vPE, to guarantee the validity of this, we make the VPT
4613 * memory coherent with the CPU caches here.
4614 */
4615 if (find_4_1_its() && !atomic_read(&vpe->vmapp_count))
4616 gic_flush_dcache_to_poc(page_address(vpe->vpt_page),
4617 LPI_PENDBASE_SZ);
4618 }
4619
4620 static const struct irq_domain_ops its_vpe_domain_ops = {
4621 .alloc = its_vpe_irq_domain_alloc,
4622 .free = its_vpe_irq_domain_free,
4623 .activate = its_vpe_irq_domain_activate,
4624 .deactivate = its_vpe_irq_domain_deactivate,
4625 };
4626
its_force_quiescent(void __iomem * base)4627 static int its_force_quiescent(void __iomem *base)
4628 {
4629 u32 count = 1000000; /* 1s */
4630 u32 val;
4631
4632 val = readl_relaxed(base + GITS_CTLR);
4633 /*
4634 * GIC architecture specification requires the ITS to be both
4635 * disabled and quiescent for writes to GITS_BASER<n> or
4636 * GITS_CBASER to not have UNPREDICTABLE results.
4637 */
4638 if ((val & GITS_CTLR_QUIESCENT) && !(val & GITS_CTLR_ENABLE))
4639 return 0;
4640
4641 /* Disable the generation of all interrupts to this ITS */
4642 val &= ~(GITS_CTLR_ENABLE | GITS_CTLR_ImDe);
4643 writel_relaxed(val, base + GITS_CTLR);
4644
4645 /* Poll GITS_CTLR and wait until ITS becomes quiescent */
4646 while (1) {
4647 val = readl_relaxed(base + GITS_CTLR);
4648 if (val & GITS_CTLR_QUIESCENT)
4649 return 0;
4650
4651 count--;
4652 if (!count)
4653 return -EBUSY;
4654
4655 cpu_relax();
4656 udelay(1);
4657 }
4658 }
4659
its_enable_quirk_cavium_22375(void * data)4660 static bool __maybe_unused its_enable_quirk_cavium_22375(void *data)
4661 {
4662 struct its_node *its = data;
4663
4664 /* erratum 22375: only alloc 8MB table size (20 bits) */
4665 its->typer &= ~GITS_TYPER_DEVBITS;
4666 its->typer |= FIELD_PREP(GITS_TYPER_DEVBITS, 20 - 1);
4667 its->flags |= ITS_FLAGS_WORKAROUND_CAVIUM_22375;
4668
4669 return true;
4670 }
4671
its_enable_quirk_cavium_23144(void * data)4672 static bool __maybe_unused its_enable_quirk_cavium_23144(void *data)
4673 {
4674 struct its_node *its = data;
4675
4676 its->flags |= ITS_FLAGS_WORKAROUND_CAVIUM_23144;
4677
4678 return true;
4679 }
4680
its_enable_quirk_qdf2400_e0065(void * data)4681 static bool __maybe_unused its_enable_quirk_qdf2400_e0065(void *data)
4682 {
4683 struct its_node *its = data;
4684
4685 /* On QDF2400, the size of the ITE is 16Bytes */
4686 its->typer &= ~GITS_TYPER_ITT_ENTRY_SIZE;
4687 its->typer |= FIELD_PREP(GITS_TYPER_ITT_ENTRY_SIZE, 16 - 1);
4688
4689 return true;
4690 }
4691
its_irq_get_msi_base_pre_its(struct its_device * its_dev)4692 static u64 its_irq_get_msi_base_pre_its(struct its_device *its_dev)
4693 {
4694 struct its_node *its = its_dev->its;
4695
4696 /*
4697 * The Socionext Synquacer SoC has a so-called 'pre-ITS',
4698 * which maps 32-bit writes targeted at a separate window of
4699 * size '4 << device_id_bits' onto writes to GITS_TRANSLATER
4700 * with device ID taken from bits [device_id_bits + 1:2] of
4701 * the window offset.
4702 */
4703 return its->pre_its_base + (its_dev->device_id << 2);
4704 }
4705
its_enable_quirk_socionext_synquacer(void * data)4706 static bool __maybe_unused its_enable_quirk_socionext_synquacer(void *data)
4707 {
4708 struct its_node *its = data;
4709 u32 pre_its_window[2];
4710 u32 ids;
4711
4712 if (!fwnode_property_read_u32_array(its->fwnode_handle,
4713 "socionext,synquacer-pre-its",
4714 pre_its_window,
4715 ARRAY_SIZE(pre_its_window))) {
4716
4717 its->pre_its_base = pre_its_window[0];
4718 its->get_msi_base = its_irq_get_msi_base_pre_its;
4719
4720 ids = ilog2(pre_its_window[1]) - 2;
4721 if (device_ids(its) > ids) {
4722 its->typer &= ~GITS_TYPER_DEVBITS;
4723 its->typer |= FIELD_PREP(GITS_TYPER_DEVBITS, ids - 1);
4724 }
4725
4726 /* the pre-ITS breaks isolation, so disable MSI remapping */
4727 its->msi_domain_flags &= ~IRQ_DOMAIN_FLAG_ISOLATED_MSI;
4728 return true;
4729 }
4730 return false;
4731 }
4732
its_enable_quirk_hip07_161600802(void * data)4733 static bool __maybe_unused its_enable_quirk_hip07_161600802(void *data)
4734 {
4735 struct its_node *its = data;
4736
4737 /*
4738 * Hip07 insists on using the wrong address for the VLPI
4739 * page. Trick it into doing the right thing...
4740 */
4741 its->vlpi_redist_offset = SZ_128K;
4742 return true;
4743 }
4744
its_enable_rk3588001(void * data)4745 static bool __maybe_unused its_enable_rk3588001(void *data)
4746 {
4747 struct its_node *its = data;
4748
4749 if (!of_machine_is_compatible("rockchip,rk3588") &&
4750 !of_machine_is_compatible("rockchip,rk3588s"))
4751 return false;
4752
4753 its->flags |= ITS_FLAGS_FORCE_NON_SHAREABLE;
4754 gic_rdists->flags |= RDIST_FLAGS_FORCE_NON_SHAREABLE;
4755
4756 return true;
4757 }
4758
its_set_non_coherent(void * data)4759 static bool its_set_non_coherent(void *data)
4760 {
4761 struct its_node *its = data;
4762
4763 its->flags |= ITS_FLAGS_FORCE_NON_SHAREABLE;
4764 return true;
4765 }
4766
4767 static const struct gic_quirk its_quirks[] = {
4768 #ifdef CONFIG_CAVIUM_ERRATUM_22375
4769 {
4770 .desc = "ITS: Cavium errata 22375, 24313",
4771 .iidr = 0xa100034c, /* ThunderX pass 1.x */
4772 .mask = 0xffff0fff,
4773 .init = its_enable_quirk_cavium_22375,
4774 },
4775 #endif
4776 #ifdef CONFIG_CAVIUM_ERRATUM_23144
4777 {
4778 .desc = "ITS: Cavium erratum 23144",
4779 .iidr = 0xa100034c, /* ThunderX pass 1.x */
4780 .mask = 0xffff0fff,
4781 .init = its_enable_quirk_cavium_23144,
4782 },
4783 #endif
4784 #ifdef CONFIG_QCOM_QDF2400_ERRATUM_0065
4785 {
4786 .desc = "ITS: QDF2400 erratum 0065",
4787 .iidr = 0x00001070, /* QDF2400 ITS rev 1.x */
4788 .mask = 0xffffffff,
4789 .init = its_enable_quirk_qdf2400_e0065,
4790 },
4791 #endif
4792 #ifdef CONFIG_SOCIONEXT_SYNQUACER_PREITS
4793 {
4794 /*
4795 * The Socionext Synquacer SoC incorporates ARM's own GIC-500
4796 * implementation, but with a 'pre-ITS' added that requires
4797 * special handling in software.
4798 */
4799 .desc = "ITS: Socionext Synquacer pre-ITS",
4800 .iidr = 0x0001143b,
4801 .mask = 0xffffffff,
4802 .init = its_enable_quirk_socionext_synquacer,
4803 },
4804 #endif
4805 #ifdef CONFIG_HISILICON_ERRATUM_161600802
4806 {
4807 .desc = "ITS: Hip07 erratum 161600802",
4808 .iidr = 0x00000004,
4809 .mask = 0xffffffff,
4810 .init = its_enable_quirk_hip07_161600802,
4811 },
4812 #endif
4813 #ifdef CONFIG_ROCKCHIP_ERRATUM_3588001
4814 {
4815 .desc = "ITS: Rockchip erratum RK3588001",
4816 .iidr = 0x0201743b,
4817 .mask = 0xffffffff,
4818 .init = its_enable_rk3588001,
4819 },
4820 #endif
4821 {
4822 .desc = "ITS: non-coherent attribute",
4823 .property = "dma-noncoherent",
4824 .init = its_set_non_coherent,
4825 },
4826 {
4827 }
4828 };
4829
its_enable_quirks(struct its_node * its)4830 static void its_enable_quirks(struct its_node *its)
4831 {
4832 u32 iidr = readl_relaxed(its->base + GITS_IIDR);
4833
4834 gic_enable_quirks(iidr, its_quirks, its);
4835
4836 if (is_of_node(its->fwnode_handle))
4837 gic_enable_of_quirks(to_of_node(its->fwnode_handle),
4838 its_quirks, its);
4839 }
4840
its_save_disable(void)4841 static int its_save_disable(void)
4842 {
4843 struct its_node *its;
4844 int err = 0;
4845
4846 raw_spin_lock(&its_lock);
4847 list_for_each_entry(its, &its_nodes, entry) {
4848 void __iomem *base;
4849
4850 base = its->base;
4851 its->ctlr_save = readl_relaxed(base + GITS_CTLR);
4852 err = its_force_quiescent(base);
4853 if (err) {
4854 pr_err("ITS@%pa: failed to quiesce: %d\n",
4855 &its->phys_base, err);
4856 writel_relaxed(its->ctlr_save, base + GITS_CTLR);
4857 goto err;
4858 }
4859
4860 its->cbaser_save = gits_read_cbaser(base + GITS_CBASER);
4861 }
4862
4863 err:
4864 if (err) {
4865 list_for_each_entry_continue_reverse(its, &its_nodes, entry) {
4866 void __iomem *base;
4867
4868 base = its->base;
4869 writel_relaxed(its->ctlr_save, base + GITS_CTLR);
4870 }
4871 }
4872 raw_spin_unlock(&its_lock);
4873
4874 return err;
4875 }
4876
its_restore_enable(void)4877 static void its_restore_enable(void)
4878 {
4879 struct its_node *its;
4880 int ret;
4881
4882 raw_spin_lock(&its_lock);
4883 list_for_each_entry(its, &its_nodes, entry) {
4884 void __iomem *base;
4885 int i;
4886
4887 base = its->base;
4888
4889 /*
4890 * Make sure that the ITS is disabled. If it fails to quiesce,
4891 * don't restore it since writing to CBASER or BASER<n>
4892 * registers is undefined according to the GIC v3 ITS
4893 * Specification.
4894 *
4895 * Firmware resuming with the ITS enabled is terminally broken.
4896 */
4897 WARN_ON(readl_relaxed(base + GITS_CTLR) & GITS_CTLR_ENABLE);
4898 ret = its_force_quiescent(base);
4899 if (ret) {
4900 pr_err("ITS@%pa: failed to quiesce on resume: %d\n",
4901 &its->phys_base, ret);
4902 continue;
4903 }
4904
4905 gits_write_cbaser(its->cbaser_save, base + GITS_CBASER);
4906
4907 /*
4908 * Writing CBASER resets CREADR to 0, so make CWRITER and
4909 * cmd_write line up with it.
4910 */
4911 its->cmd_write = its->cmd_base;
4912 gits_write_cwriter(0, base + GITS_CWRITER);
4913
4914 /* Restore GITS_BASER from the value cache. */
4915 for (i = 0; i < GITS_BASER_NR_REGS; i++) {
4916 struct its_baser *baser = &its->tables[i];
4917
4918 if (!(baser->val & GITS_BASER_VALID))
4919 continue;
4920
4921 its_write_baser(its, baser, baser->val);
4922 }
4923 writel_relaxed(its->ctlr_save, base + GITS_CTLR);
4924
4925 /*
4926 * Reinit the collection if it's stored in the ITS. This is
4927 * indicated by the col_id being less than the HCC field.
4928 * CID < HCC as specified in the GIC v3 Documentation.
4929 */
4930 if (its->collections[smp_processor_id()].col_id <
4931 GITS_TYPER_HCC(gic_read_typer(base + GITS_TYPER)))
4932 its_cpu_init_collection(its);
4933 }
4934 raw_spin_unlock(&its_lock);
4935 }
4936
4937 static struct syscore_ops its_syscore_ops = {
4938 .suspend = its_save_disable,
4939 .resume = its_restore_enable,
4940 };
4941
its_map_one(struct resource * res,int * err)4942 static void __init __iomem *its_map_one(struct resource *res, int *err)
4943 {
4944 void __iomem *its_base;
4945 u32 val;
4946
4947 its_base = ioremap(res->start, SZ_64K);
4948 if (!its_base) {
4949 pr_warn("ITS@%pa: Unable to map ITS registers\n", &res->start);
4950 *err = -ENOMEM;
4951 return NULL;
4952 }
4953
4954 val = readl_relaxed(its_base + GITS_PIDR2) & GIC_PIDR2_ARCH_MASK;
4955 if (val != 0x30 && val != 0x40) {
4956 pr_warn("ITS@%pa: No ITS detected, giving up\n", &res->start);
4957 *err = -ENODEV;
4958 goto out_unmap;
4959 }
4960
4961 *err = its_force_quiescent(its_base);
4962 if (*err) {
4963 pr_warn("ITS@%pa: Failed to quiesce, giving up\n", &res->start);
4964 goto out_unmap;
4965 }
4966
4967 return its_base;
4968
4969 out_unmap:
4970 iounmap(its_base);
4971 return NULL;
4972 }
4973
its_init_domain(struct its_node * its)4974 static int its_init_domain(struct its_node *its)
4975 {
4976 struct irq_domain *inner_domain;
4977 struct msi_domain_info *info;
4978
4979 info = kzalloc(sizeof(*info), GFP_KERNEL);
4980 if (!info)
4981 return -ENOMEM;
4982
4983 info->ops = &its_msi_domain_ops;
4984 info->data = its;
4985
4986 inner_domain = irq_domain_create_hierarchy(its_parent,
4987 its->msi_domain_flags, 0,
4988 its->fwnode_handle, &its_domain_ops,
4989 info);
4990 if (!inner_domain) {
4991 kfree(info);
4992 return -ENOMEM;
4993 }
4994
4995 irq_domain_update_bus_token(inner_domain, DOMAIN_BUS_NEXUS);
4996
4997 return 0;
4998 }
4999
its_init_vpe_domain(void)5000 static int its_init_vpe_domain(void)
5001 {
5002 struct its_node *its;
5003 u32 devid;
5004 int entries;
5005
5006 if (gic_rdists->has_direct_lpi) {
5007 pr_info("ITS: Using DirectLPI for VPE invalidation\n");
5008 return 0;
5009 }
5010
5011 /* Any ITS will do, even if not v4 */
5012 its = list_first_entry(&its_nodes, struct its_node, entry);
5013
5014 entries = roundup_pow_of_two(nr_cpu_ids);
5015 vpe_proxy.vpes = kcalloc(entries, sizeof(*vpe_proxy.vpes),
5016 GFP_KERNEL);
5017 if (!vpe_proxy.vpes)
5018 return -ENOMEM;
5019
5020 /* Use the last possible DevID */
5021 devid = GENMASK(device_ids(its) - 1, 0);
5022 vpe_proxy.dev = its_create_device(its, devid, entries, false);
5023 if (!vpe_proxy.dev) {
5024 kfree(vpe_proxy.vpes);
5025 pr_err("ITS: Can't allocate GICv4 proxy device\n");
5026 return -ENOMEM;
5027 }
5028
5029 BUG_ON(entries > vpe_proxy.dev->nr_ites);
5030
5031 raw_spin_lock_init(&vpe_proxy.lock);
5032 vpe_proxy.next_victim = 0;
5033 pr_info("ITS: Allocated DevID %x as GICv4 proxy device (%d slots)\n",
5034 devid, vpe_proxy.dev->nr_ites);
5035
5036 return 0;
5037 }
5038
its_compute_its_list_map(struct its_node * its)5039 static int __init its_compute_its_list_map(struct its_node *its)
5040 {
5041 int its_number;
5042 u32 ctlr;
5043
5044 /*
5045 * This is assumed to be done early enough that we're
5046 * guaranteed to be single-threaded, hence no
5047 * locking. Should this change, we should address
5048 * this.
5049 */
5050 its_number = find_first_zero_bit(&its_list_map, GICv4_ITS_LIST_MAX);
5051 if (its_number >= GICv4_ITS_LIST_MAX) {
5052 pr_err("ITS@%pa: No ITSList entry available!\n",
5053 &its->phys_base);
5054 return -EINVAL;
5055 }
5056
5057 ctlr = readl_relaxed(its->base + GITS_CTLR);
5058 ctlr &= ~GITS_CTLR_ITS_NUMBER;
5059 ctlr |= its_number << GITS_CTLR_ITS_NUMBER_SHIFT;
5060 writel_relaxed(ctlr, its->base + GITS_CTLR);
5061 ctlr = readl_relaxed(its->base + GITS_CTLR);
5062 if ((ctlr & GITS_CTLR_ITS_NUMBER) != (its_number << GITS_CTLR_ITS_NUMBER_SHIFT)) {
5063 its_number = ctlr & GITS_CTLR_ITS_NUMBER;
5064 its_number >>= GITS_CTLR_ITS_NUMBER_SHIFT;
5065 }
5066
5067 if (test_and_set_bit(its_number, &its_list_map)) {
5068 pr_err("ITS@%pa: Duplicate ITSList entry %d\n",
5069 &its->phys_base, its_number);
5070 return -EINVAL;
5071 }
5072
5073 return its_number;
5074 }
5075
its_probe_one(struct its_node * its)5076 static int __init its_probe_one(struct its_node *its)
5077 {
5078 u64 baser, tmp;
5079 struct page *page;
5080 u32 ctlr;
5081 int err;
5082
5083 its_enable_quirks(its);
5084
5085 if (is_v4(its)) {
5086 if (!(its->typer & GITS_TYPER_VMOVP)) {
5087 err = its_compute_its_list_map(its);
5088 if (err < 0)
5089 goto out;
5090
5091 its->list_nr = err;
5092
5093 pr_info("ITS@%pa: Using ITS number %d\n",
5094 &its->phys_base, err);
5095 } else {
5096 pr_info("ITS@%pa: Single VMOVP capable\n", &its->phys_base);
5097 }
5098
5099 if (is_v4_1(its)) {
5100 u32 svpet = FIELD_GET(GITS_TYPER_SVPET, its->typer);
5101
5102 its->sgir_base = ioremap(its->phys_base + SZ_128K, SZ_64K);
5103 if (!its->sgir_base) {
5104 err = -ENOMEM;
5105 goto out;
5106 }
5107
5108 its->mpidr = readl_relaxed(its->base + GITS_MPIDR);
5109
5110 pr_info("ITS@%pa: Using GICv4.1 mode %08x %08x\n",
5111 &its->phys_base, its->mpidr, svpet);
5112 }
5113 }
5114
5115 page = alloc_pages_node(its->numa_node, GFP_KERNEL | __GFP_ZERO,
5116 get_order(ITS_CMD_QUEUE_SZ));
5117 if (!page) {
5118 err = -ENOMEM;
5119 goto out_unmap_sgir;
5120 }
5121 its->cmd_base = (void *)page_address(page);
5122 its->cmd_write = its->cmd_base;
5123
5124 err = its_alloc_tables(its);
5125 if (err)
5126 goto out_free_cmd;
5127
5128 err = its_alloc_collections(its);
5129 if (err)
5130 goto out_free_tables;
5131
5132 baser = (virt_to_phys(its->cmd_base) |
5133 GITS_CBASER_RaWaWb |
5134 GITS_CBASER_InnerShareable |
5135 (ITS_CMD_QUEUE_SZ / SZ_4K - 1) |
5136 GITS_CBASER_VALID);
5137
5138 gits_write_cbaser(baser, its->base + GITS_CBASER);
5139 tmp = gits_read_cbaser(its->base + GITS_CBASER);
5140
5141 if (its->flags & ITS_FLAGS_FORCE_NON_SHAREABLE)
5142 tmp &= ~GITS_CBASER_SHAREABILITY_MASK;
5143
5144 if ((tmp ^ baser) & GITS_CBASER_SHAREABILITY_MASK) {
5145 if (!(tmp & GITS_CBASER_SHAREABILITY_MASK)) {
5146 /*
5147 * The HW reports non-shareable, we must
5148 * remove the cacheability attributes as
5149 * well.
5150 */
5151 baser &= ~(GITS_CBASER_SHAREABILITY_MASK |
5152 GITS_CBASER_CACHEABILITY_MASK);
5153 baser |= GITS_CBASER_nC;
5154 gits_write_cbaser(baser, its->base + GITS_CBASER);
5155 }
5156 pr_info("ITS: using cache flushing for cmd queue\n");
5157 its->flags |= ITS_FLAGS_CMDQ_NEEDS_FLUSHING;
5158 }
5159
5160 gits_write_cwriter(0, its->base + GITS_CWRITER);
5161 ctlr = readl_relaxed(its->base + GITS_CTLR);
5162 ctlr |= GITS_CTLR_ENABLE;
5163 if (is_v4(its))
5164 ctlr |= GITS_CTLR_ImDe;
5165 writel_relaxed(ctlr, its->base + GITS_CTLR);
5166
5167 err = its_init_domain(its);
5168 if (err)
5169 goto out_free_tables;
5170
5171 raw_spin_lock(&its_lock);
5172 list_add(&its->entry, &its_nodes);
5173 raw_spin_unlock(&its_lock);
5174
5175 return 0;
5176
5177 out_free_tables:
5178 its_free_tables(its);
5179 out_free_cmd:
5180 free_pages((unsigned long)its->cmd_base, get_order(ITS_CMD_QUEUE_SZ));
5181 out_unmap_sgir:
5182 if (its->sgir_base)
5183 iounmap(its->sgir_base);
5184 out:
5185 pr_err("ITS@%pa: failed probing (%d)\n", &its->phys_base, err);
5186 return err;
5187 }
5188
gic_rdists_supports_plpis(void)5189 static bool gic_rdists_supports_plpis(void)
5190 {
5191 return !!(gic_read_typer(gic_data_rdist_rd_base() + GICR_TYPER) & GICR_TYPER_PLPIS);
5192 }
5193
redist_disable_lpis(void)5194 static int redist_disable_lpis(void)
5195 {
5196 void __iomem *rbase = gic_data_rdist_rd_base();
5197 u64 timeout = USEC_PER_SEC;
5198 u64 val;
5199
5200 if (!gic_rdists_supports_plpis()) {
5201 pr_info("CPU%d: LPIs not supported\n", smp_processor_id());
5202 return -ENXIO;
5203 }
5204
5205 val = readl_relaxed(rbase + GICR_CTLR);
5206 if (!(val & GICR_CTLR_ENABLE_LPIS))
5207 return 0;
5208
5209 /*
5210 * If coming via a CPU hotplug event, we don't need to disable
5211 * LPIs before trying to re-enable them. They are already
5212 * configured and all is well in the world.
5213 *
5214 * If running with preallocated tables, there is nothing to do.
5215 */
5216 if ((gic_data_rdist()->flags & RD_LOCAL_LPI_ENABLED) ||
5217 (gic_rdists->flags & RDIST_FLAGS_RD_TABLES_PREALLOCATED))
5218 return 0;
5219
5220 /*
5221 * From that point on, we only try to do some damage control.
5222 */
5223 pr_warn("GICv3: CPU%d: Booted with LPIs enabled, memory probably corrupted\n",
5224 smp_processor_id());
5225 add_taint(TAINT_CRAP, LOCKDEP_STILL_OK);
5226
5227 /* Disable LPIs */
5228 val &= ~GICR_CTLR_ENABLE_LPIS;
5229 writel_relaxed(val, rbase + GICR_CTLR);
5230
5231 /* Make sure any change to GICR_CTLR is observable by the GIC */
5232 dsb(sy);
5233
5234 /*
5235 * Software must observe RWP==0 after clearing GICR_CTLR.EnableLPIs
5236 * from 1 to 0 before programming GICR_PEND{PROP}BASER registers.
5237 * Error out if we time out waiting for RWP to clear.
5238 */
5239 while (readl_relaxed(rbase + GICR_CTLR) & GICR_CTLR_RWP) {
5240 if (!timeout) {
5241 pr_err("CPU%d: Timeout while disabling LPIs\n",
5242 smp_processor_id());
5243 return -ETIMEDOUT;
5244 }
5245 udelay(1);
5246 timeout--;
5247 }
5248
5249 /*
5250 * After it has been written to 1, it is IMPLEMENTATION
5251 * DEFINED whether GICR_CTLR.EnableLPI becomes RES1 or can be
5252 * cleared to 0. Error out if clearing the bit failed.
5253 */
5254 if (readl_relaxed(rbase + GICR_CTLR) & GICR_CTLR_ENABLE_LPIS) {
5255 pr_err("CPU%d: Failed to disable LPIs\n", smp_processor_id());
5256 return -EBUSY;
5257 }
5258
5259 return 0;
5260 }
5261
its_cpu_init(void)5262 int its_cpu_init(void)
5263 {
5264 if (!list_empty(&its_nodes)) {
5265 int ret;
5266
5267 ret = redist_disable_lpis();
5268 if (ret)
5269 return ret;
5270
5271 its_cpu_init_lpis();
5272 its_cpu_init_collections();
5273 }
5274
5275 return 0;
5276 }
5277
rdist_memreserve_cpuhp_cleanup_workfn(struct work_struct * work)5278 static void rdist_memreserve_cpuhp_cleanup_workfn(struct work_struct *work)
5279 {
5280 cpuhp_remove_state_nocalls(gic_rdists->cpuhp_memreserve_state);
5281 gic_rdists->cpuhp_memreserve_state = CPUHP_INVALID;
5282 }
5283
5284 static DECLARE_WORK(rdist_memreserve_cpuhp_cleanup_work,
5285 rdist_memreserve_cpuhp_cleanup_workfn);
5286
its_cpu_memreserve_lpi(unsigned int cpu)5287 static int its_cpu_memreserve_lpi(unsigned int cpu)
5288 {
5289 struct page *pend_page;
5290 int ret = 0;
5291
5292 /* This gets to run exactly once per CPU */
5293 if (gic_data_rdist()->flags & RD_LOCAL_MEMRESERVE_DONE)
5294 return 0;
5295
5296 pend_page = gic_data_rdist()->pend_page;
5297 if (WARN_ON(!pend_page)) {
5298 ret = -ENOMEM;
5299 goto out;
5300 }
5301 /*
5302 * If the pending table was pre-programmed, free the memory we
5303 * preemptively allocated. Otherwise, reserve that memory for
5304 * later kexecs.
5305 */
5306 if (gic_data_rdist()->flags & RD_LOCAL_PENDTABLE_PREALLOCATED) {
5307 its_free_pending_table(pend_page);
5308 gic_data_rdist()->pend_page = NULL;
5309 } else {
5310 phys_addr_t paddr = page_to_phys(pend_page);
5311 WARN_ON(gic_reserve_range(paddr, LPI_PENDBASE_SZ));
5312 }
5313
5314 out:
5315 /* Last CPU being brought up gets to issue the cleanup */
5316 if (!IS_ENABLED(CONFIG_SMP) ||
5317 cpumask_equal(&cpus_booted_once_mask, cpu_possible_mask))
5318 schedule_work(&rdist_memreserve_cpuhp_cleanup_work);
5319
5320 gic_data_rdist()->flags |= RD_LOCAL_MEMRESERVE_DONE;
5321 return ret;
5322 }
5323
5324 /* Mark all the BASER registers as invalid before they get reprogrammed */
its_reset_one(struct resource * res)5325 static int __init its_reset_one(struct resource *res)
5326 {
5327 void __iomem *its_base;
5328 int err, i;
5329
5330 its_base = its_map_one(res, &err);
5331 if (!its_base)
5332 return err;
5333
5334 for (i = 0; i < GITS_BASER_NR_REGS; i++)
5335 gits_write_baser(0, its_base + GITS_BASER + (i << 3));
5336
5337 iounmap(its_base);
5338 return 0;
5339 }
5340
5341 static const struct of_device_id its_device_id[] = {
5342 { .compatible = "arm,gic-v3-its", },
5343 {},
5344 };
5345
its_node_init(struct resource * res,struct fwnode_handle * handle,int numa_node)5346 static struct its_node __init *its_node_init(struct resource *res,
5347 struct fwnode_handle *handle, int numa_node)
5348 {
5349 void __iomem *its_base;
5350 struct its_node *its;
5351 int err;
5352
5353 its_base = its_map_one(res, &err);
5354 if (!its_base)
5355 return NULL;
5356
5357 pr_info("ITS %pR\n", res);
5358
5359 its = kzalloc(sizeof(*its), GFP_KERNEL);
5360 if (!its)
5361 goto out_unmap;
5362
5363 raw_spin_lock_init(&its->lock);
5364 mutex_init(&its->dev_alloc_lock);
5365 INIT_LIST_HEAD(&its->entry);
5366 INIT_LIST_HEAD(&its->its_device_list);
5367
5368 its->typer = gic_read_typer(its_base + GITS_TYPER);
5369 its->base = its_base;
5370 its->phys_base = res->start;
5371 its->get_msi_base = its_irq_get_msi_base;
5372 its->msi_domain_flags = IRQ_DOMAIN_FLAG_ISOLATED_MSI;
5373
5374 its->numa_node = numa_node;
5375 its->fwnode_handle = handle;
5376
5377 return its;
5378
5379 out_unmap:
5380 iounmap(its_base);
5381 return NULL;
5382 }
5383
its_node_destroy(struct its_node * its)5384 static void its_node_destroy(struct its_node *its)
5385 {
5386 iounmap(its->base);
5387 kfree(its);
5388 }
5389
its_of_probe(struct device_node * node)5390 static int __init its_of_probe(struct device_node *node)
5391 {
5392 struct device_node *np;
5393 struct resource res;
5394 int err;
5395
5396 /*
5397 * Make sure *all* the ITS are reset before we probe any, as
5398 * they may be sharing memory. If any of the ITS fails to
5399 * reset, don't even try to go any further, as this could
5400 * result in something even worse.
5401 */
5402 for (np = of_find_matching_node(node, its_device_id); np;
5403 np = of_find_matching_node(np, its_device_id)) {
5404 if (!of_device_is_available(np) ||
5405 !of_property_read_bool(np, "msi-controller") ||
5406 of_address_to_resource(np, 0, &res))
5407 continue;
5408
5409 err = its_reset_one(&res);
5410 if (err)
5411 return err;
5412 }
5413
5414 for (np = of_find_matching_node(node, its_device_id); np;
5415 np = of_find_matching_node(np, its_device_id)) {
5416 struct its_node *its;
5417
5418 if (!of_device_is_available(np))
5419 continue;
5420 if (!of_property_read_bool(np, "msi-controller")) {
5421 pr_warn("%pOF: no msi-controller property, ITS ignored\n",
5422 np);
5423 continue;
5424 }
5425
5426 if (of_address_to_resource(np, 0, &res)) {
5427 pr_warn("%pOF: no regs?\n", np);
5428 continue;
5429 }
5430
5431
5432 its = its_node_init(&res, &np->fwnode, of_node_to_nid(np));
5433 if (!its)
5434 return -ENOMEM;
5435
5436 err = its_probe_one(its);
5437 if (err) {
5438 its_node_destroy(its);
5439 return err;
5440 }
5441 }
5442 return 0;
5443 }
5444
5445 #ifdef CONFIG_ACPI
5446
5447 #define ACPI_GICV3_ITS_MEM_SIZE (SZ_128K)
5448
5449 #ifdef CONFIG_ACPI_NUMA
5450 struct its_srat_map {
5451 /* numa node id */
5452 u32 numa_node;
5453 /* GIC ITS ID */
5454 u32 its_id;
5455 };
5456
5457 static struct its_srat_map *its_srat_maps __initdata;
5458 static int its_in_srat __initdata;
5459
acpi_get_its_numa_node(u32 its_id)5460 static int __init acpi_get_its_numa_node(u32 its_id)
5461 {
5462 int i;
5463
5464 for (i = 0; i < its_in_srat; i++) {
5465 if (its_id == its_srat_maps[i].its_id)
5466 return its_srat_maps[i].numa_node;
5467 }
5468 return NUMA_NO_NODE;
5469 }
5470
gic_acpi_match_srat_its(union acpi_subtable_headers * header,const unsigned long end)5471 static int __init gic_acpi_match_srat_its(union acpi_subtable_headers *header,
5472 const unsigned long end)
5473 {
5474 return 0;
5475 }
5476
gic_acpi_parse_srat_its(union acpi_subtable_headers * header,const unsigned long end)5477 static int __init gic_acpi_parse_srat_its(union acpi_subtable_headers *header,
5478 const unsigned long end)
5479 {
5480 int node;
5481 struct acpi_srat_gic_its_affinity *its_affinity;
5482
5483 its_affinity = (struct acpi_srat_gic_its_affinity *)header;
5484 if (!its_affinity)
5485 return -EINVAL;
5486
5487 if (its_affinity->header.length < sizeof(*its_affinity)) {
5488 pr_err("SRAT: Invalid header length %d in ITS affinity\n",
5489 its_affinity->header.length);
5490 return -EINVAL;
5491 }
5492
5493 /*
5494 * Note that in theory a new proximity node could be created by this
5495 * entry as it is an SRAT resource allocation structure.
5496 * We do not currently support doing so.
5497 */
5498 node = pxm_to_node(its_affinity->proximity_domain);
5499
5500 if (node == NUMA_NO_NODE || node >= MAX_NUMNODES) {
5501 pr_err("SRAT: Invalid NUMA node %d in ITS affinity\n", node);
5502 return 0;
5503 }
5504
5505 its_srat_maps[its_in_srat].numa_node = node;
5506 its_srat_maps[its_in_srat].its_id = its_affinity->its_id;
5507 its_in_srat++;
5508 pr_info("SRAT: PXM %d -> ITS %d -> Node %d\n",
5509 its_affinity->proximity_domain, its_affinity->its_id, node);
5510
5511 return 0;
5512 }
5513
acpi_table_parse_srat_its(void)5514 static void __init acpi_table_parse_srat_its(void)
5515 {
5516 int count;
5517
5518 count = acpi_table_parse_entries(ACPI_SIG_SRAT,
5519 sizeof(struct acpi_table_srat),
5520 ACPI_SRAT_TYPE_GIC_ITS_AFFINITY,
5521 gic_acpi_match_srat_its, 0);
5522 if (count <= 0)
5523 return;
5524
5525 its_srat_maps = kmalloc_array(count, sizeof(struct its_srat_map),
5526 GFP_KERNEL);
5527 if (!its_srat_maps)
5528 return;
5529
5530 acpi_table_parse_entries(ACPI_SIG_SRAT,
5531 sizeof(struct acpi_table_srat),
5532 ACPI_SRAT_TYPE_GIC_ITS_AFFINITY,
5533 gic_acpi_parse_srat_its, 0);
5534 }
5535
5536 /* free the its_srat_maps after ITS probing */
acpi_its_srat_maps_free(void)5537 static void __init acpi_its_srat_maps_free(void)
5538 {
5539 kfree(its_srat_maps);
5540 }
5541 #else
acpi_table_parse_srat_its(void)5542 static void __init acpi_table_parse_srat_its(void) { }
acpi_get_its_numa_node(u32 its_id)5543 static int __init acpi_get_its_numa_node(u32 its_id) { return NUMA_NO_NODE; }
acpi_its_srat_maps_free(void)5544 static void __init acpi_its_srat_maps_free(void) { }
5545 #endif
5546
gic_acpi_parse_madt_its(union acpi_subtable_headers * header,const unsigned long end)5547 static int __init gic_acpi_parse_madt_its(union acpi_subtable_headers *header,
5548 const unsigned long end)
5549 {
5550 struct acpi_madt_generic_translator *its_entry;
5551 struct fwnode_handle *dom_handle;
5552 struct its_node *its;
5553 struct resource res;
5554 int err;
5555
5556 its_entry = (struct acpi_madt_generic_translator *)header;
5557 memset(&res, 0, sizeof(res));
5558 res.start = its_entry->base_address;
5559 res.end = its_entry->base_address + ACPI_GICV3_ITS_MEM_SIZE - 1;
5560 res.flags = IORESOURCE_MEM;
5561
5562 dom_handle = irq_domain_alloc_fwnode(&res.start);
5563 if (!dom_handle) {
5564 pr_err("ITS@%pa: Unable to allocate GICv3 ITS domain token\n",
5565 &res.start);
5566 return -ENOMEM;
5567 }
5568
5569 err = iort_register_domain_token(its_entry->translation_id, res.start,
5570 dom_handle);
5571 if (err) {
5572 pr_err("ITS@%pa: Unable to register GICv3 ITS domain token (ITS ID %d) to IORT\n",
5573 &res.start, its_entry->translation_id);
5574 goto dom_err;
5575 }
5576
5577 its = its_node_init(&res, dom_handle,
5578 acpi_get_its_numa_node(its_entry->translation_id));
5579 if (!its) {
5580 err = -ENOMEM;
5581 goto node_err;
5582 }
5583
5584 err = its_probe_one(its);
5585 if (!err)
5586 return 0;
5587
5588 node_err:
5589 iort_deregister_domain_token(its_entry->translation_id);
5590 dom_err:
5591 irq_domain_free_fwnode(dom_handle);
5592 return err;
5593 }
5594
its_acpi_reset(union acpi_subtable_headers * header,const unsigned long end)5595 static int __init its_acpi_reset(union acpi_subtable_headers *header,
5596 const unsigned long end)
5597 {
5598 struct acpi_madt_generic_translator *its_entry;
5599 struct resource res;
5600
5601 its_entry = (struct acpi_madt_generic_translator *)header;
5602 res = (struct resource) {
5603 .start = its_entry->base_address,
5604 .end = its_entry->base_address + ACPI_GICV3_ITS_MEM_SIZE - 1,
5605 .flags = IORESOURCE_MEM,
5606 };
5607
5608 return its_reset_one(&res);
5609 }
5610
its_acpi_probe(void)5611 static void __init its_acpi_probe(void)
5612 {
5613 acpi_table_parse_srat_its();
5614 /*
5615 * Make sure *all* the ITS are reset before we probe any, as
5616 * they may be sharing memory. If any of the ITS fails to
5617 * reset, don't even try to go any further, as this could
5618 * result in something even worse.
5619 */
5620 if (acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_TRANSLATOR,
5621 its_acpi_reset, 0) > 0)
5622 acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_TRANSLATOR,
5623 gic_acpi_parse_madt_its, 0);
5624 acpi_its_srat_maps_free();
5625 }
5626 #else
its_acpi_probe(void)5627 static void __init its_acpi_probe(void) { }
5628 #endif
5629
its_lpi_memreserve_init(void)5630 int __init its_lpi_memreserve_init(void)
5631 {
5632 int state;
5633
5634 if (!efi_enabled(EFI_CONFIG_TABLES))
5635 return 0;
5636
5637 if (list_empty(&its_nodes))
5638 return 0;
5639
5640 gic_rdists->cpuhp_memreserve_state = CPUHP_INVALID;
5641 state = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN,
5642 "irqchip/arm/gicv3/memreserve:online",
5643 its_cpu_memreserve_lpi,
5644 NULL);
5645 if (state < 0)
5646 return state;
5647
5648 gic_rdists->cpuhp_memreserve_state = state;
5649
5650 return 0;
5651 }
5652
its_init(struct fwnode_handle * handle,struct rdists * rdists,struct irq_domain * parent_domain)5653 int __init its_init(struct fwnode_handle *handle, struct rdists *rdists,
5654 struct irq_domain *parent_domain)
5655 {
5656 struct device_node *of_node;
5657 struct its_node *its;
5658 bool has_v4 = false;
5659 bool has_v4_1 = false;
5660 int err;
5661
5662 gic_rdists = rdists;
5663
5664 its_parent = parent_domain;
5665 of_node = to_of_node(handle);
5666 if (of_node)
5667 its_of_probe(of_node);
5668 else
5669 its_acpi_probe();
5670
5671 if (list_empty(&its_nodes)) {
5672 pr_warn("ITS: No ITS available, not enabling LPIs\n");
5673 return -ENXIO;
5674 }
5675
5676 err = allocate_lpi_tables();
5677 if (err)
5678 return err;
5679
5680 list_for_each_entry(its, &its_nodes, entry) {
5681 has_v4 |= is_v4(its);
5682 has_v4_1 |= is_v4_1(its);
5683 }
5684
5685 /* Don't bother with inconsistent systems */
5686 if (WARN_ON(!has_v4_1 && rdists->has_rvpeid))
5687 rdists->has_rvpeid = false;
5688
5689 if (has_v4 & rdists->has_vlpis) {
5690 const struct irq_domain_ops *sgi_ops;
5691
5692 if (has_v4_1)
5693 sgi_ops = &its_sgi_domain_ops;
5694 else
5695 sgi_ops = NULL;
5696
5697 if (its_init_vpe_domain() ||
5698 its_init_v4(parent_domain, &its_vpe_domain_ops, sgi_ops)) {
5699 rdists->has_vlpis = false;
5700 pr_err("ITS: Disabling GICv4 support\n");
5701 }
5702 }
5703
5704 register_syscore_ops(&its_syscore_ops);
5705
5706 return 0;
5707 }
5708