1caab277bSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only
2cc2d3216SMarc Zyngier /*
3d7276b80SMarc Zyngier * Copyright (C) 2013-2017 ARM Limited, All Rights Reserved.
4cc2d3216SMarc Zyngier * Author: Marc Zyngier <marc.zyngier@arm.com>
5cc2d3216SMarc Zyngier */
6cc2d3216SMarc Zyngier
73f010cf1STomasz Nowicki #include <linux/acpi.h>
88d3554b8SHanjun Guo #include <linux/acpi_iort.h>
9ffedbf0cSMarc Zyngier #include <linux/bitfield.h>
10cc2d3216SMarc Zyngier #include <linux/bitmap.h>
11cc2d3216SMarc Zyngier #include <linux/cpu.h>
12c6e2ccb6SMarc Zyngier #include <linux/crash_dump.h>
13cc2d3216SMarc Zyngier #include <linux/delay.h>
143fb68faeSMarc Zyngier #include <linux/efi.h>
15cc2d3216SMarc Zyngier #include <linux/interrupt.h>
16fa49364cSRobin Murphy #include <linux/iommu.h>
1796806229SMarc Zyngier #include <linux/iopoll.h>
183f010cf1STomasz Nowicki #include <linux/irqdomain.h>
19880cb3cdSMarc Zyngier #include <linux/list.h>
20cc2d3216SMarc Zyngier #include <linux/log2.h>
215e2c9f9aSMarc Zyngier #include <linux/memblock.h>
22cc2d3216SMarc Zyngier #include <linux/mm.h>
23cc2d3216SMarc Zyngier #include <linux/msi.h>
24cc2d3216SMarc Zyngier #include <linux/of.h>
25cc2d3216SMarc Zyngier #include <linux/of_address.h>
26cc2d3216SMarc Zyngier #include <linux/of_irq.h>
27cc2d3216SMarc Zyngier #include <linux/of_pci.h>
28cc2d3216SMarc Zyngier #include <linux/of_platform.h>
29cc2d3216SMarc Zyngier #include <linux/percpu.h>
30cc2d3216SMarc Zyngier #include <linux/slab.h>
31dba0bc7bSDerek Basehore #include <linux/syscore_ops.h>
32cc2d3216SMarc Zyngier
3341a83e06SJoel Porquet #include <linux/irqchip.h>
34cc2d3216SMarc Zyngier #include <linux/irqchip/arm-gic-v3.h>
35c808eea8SMarc Zyngier #include <linux/irqchip/arm-gic-v4.h>
36cc2d3216SMarc Zyngier
37cc2d3216SMarc Zyngier #include <asm/cputype.h>
38cc2d3216SMarc Zyngier #include <asm/exception.h>
39cc2d3216SMarc Zyngier
4067510ccaSRobert Richter #include "irq-gic-common.h"
4167510ccaSRobert Richter
4294100970SRobert Richter #define ITS_FLAGS_CMDQ_NEEDS_FLUSHING (1ULL << 0)
4394100970SRobert Richter #define ITS_FLAGS_WORKAROUND_CAVIUM_22375 (1ULL << 1)
44fbf8f40eSGanapatrao Kulkarni #define ITS_FLAGS_WORKAROUND_CAVIUM_23144 (1ULL << 2)
45a8707f55SSebastian Reichel #define ITS_FLAGS_FORCE_NON_SHAREABLE (1ULL << 3)
46cc2d3216SMarc Zyngier
47c0cdc890SValentin Schneider #define RD_LOCAL_LPI_ENABLED BIT(0)
48d23bc2bcSValentin Schneider #define RD_LOCAL_PENDTABLE_PREALLOCATED BIT(1)
49d23bc2bcSValentin Schneider #define RD_LOCAL_MEMRESERVE_DONE BIT(2)
50c0cdc890SValentin Schneider
51a13b0404SMarc Zyngier static u32 lpi_id_bits;
52a13b0404SMarc Zyngier
53a13b0404SMarc Zyngier /*
54a13b0404SMarc Zyngier * We allocate memory for PROPBASE to cover 2 ^ lpi_id_bits LPIs to
55a13b0404SMarc Zyngier * deal with (one configuration byte per interrupt). PENDBASE has to
56a13b0404SMarc Zyngier * be 64kB aligned (one bit per LPI, plus 8192 bits for SPI/PPI/SGI).
57a13b0404SMarc Zyngier */
58a13b0404SMarc Zyngier #define LPI_NRBITS lpi_id_bits
59a13b0404SMarc Zyngier #define LPI_PROPBASE_SZ ALIGN(BIT(LPI_NRBITS), SZ_64K)
60a13b0404SMarc Zyngier #define LPI_PENDBASE_SZ ALIGN(BIT(LPI_NRBITS) / 8, SZ_64K)
61a13b0404SMarc Zyngier
622130b789SJulien Thierry #define LPI_PROP_DEFAULT_PRIO GICD_INT_DEF_PRI
63a13b0404SMarc Zyngier
64cc2d3216SMarc Zyngier /*
65cc2d3216SMarc Zyngier * Collection structure - just an ID, and a redistributor address to
66cc2d3216SMarc Zyngier * ping. We use one per CPU as a bag of interrupts assigned to this
67cc2d3216SMarc Zyngier * CPU.
68cc2d3216SMarc Zyngier */
69cc2d3216SMarc Zyngier struct its_collection {
70cc2d3216SMarc Zyngier u64 target_address;
71cc2d3216SMarc Zyngier u16 col_id;
72cc2d3216SMarc Zyngier };
73cc2d3216SMarc Zyngier
74cc2d3216SMarc Zyngier /*
759347359aSShanker Donthineni * The ITS_BASER structure - contains memory information, cached
769347359aSShanker Donthineni * value of BASER register configuration and ITS page size.
77466b7d16SShanker Donthineni */
78466b7d16SShanker Donthineni struct its_baser {
79466b7d16SShanker Donthineni void *base;
80466b7d16SShanker Donthineni u64 val;
81466b7d16SShanker Donthineni u32 order;
829347359aSShanker Donthineni u32 psz;
83466b7d16SShanker Donthineni };
84466b7d16SShanker Donthineni
85558b0165SArd Biesheuvel struct its_device;
86558b0165SArd Biesheuvel
87466b7d16SShanker Donthineni /*
88cc2d3216SMarc Zyngier * The ITS structure - contains most of the infrastructure, with the
89841514abSMarc Zyngier * top-level MSI domain, the command queue, the collections, and the
90841514abSMarc Zyngier * list of devices writing to it.
919791ec7dSMarc Zyngier *
929791ec7dSMarc Zyngier * dev_alloc_lock has to be taken for device allocations, while the
939791ec7dSMarc Zyngier * spinlock must be taken to parse data structures such as the device
949791ec7dSMarc Zyngier * list.
95cc2d3216SMarc Zyngier */
96cc2d3216SMarc Zyngier struct its_node {
97cc2d3216SMarc Zyngier raw_spinlock_t lock;
989791ec7dSMarc Zyngier struct mutex dev_alloc_lock;
99cc2d3216SMarc Zyngier struct list_head entry;
100cc2d3216SMarc Zyngier void __iomem *base;
1015e46a484SMarc Zyngier void __iomem *sgir_base;
102db40f0a7STomasz Nowicki phys_addr_t phys_base;
103cc2d3216SMarc Zyngier struct its_cmd_block *cmd_base;
104cc2d3216SMarc Zyngier struct its_cmd_block *cmd_write;
105466b7d16SShanker Donthineni struct its_baser tables[GITS_BASER_NR_REGS];
106cc2d3216SMarc Zyngier struct its_collection *collections;
107558b0165SArd Biesheuvel struct fwnode_handle *fwnode_handle;
108558b0165SArd Biesheuvel u64 (*get_msi_base)(struct its_device *its_dev);
1090dd57fedSMarc Zyngier u64 typer;
110dba0bc7bSDerek Basehore u64 cbaser_save;
111dba0bc7bSDerek Basehore u32 ctlr_save;
1125e516846SMarc Zyngier u32 mpidr;
113cc2d3216SMarc Zyngier struct list_head its_device_list;
114cc2d3216SMarc Zyngier u64 flags;
115debf6d02SMarc Zyngier unsigned long list_nr;
116fbf8f40eSGanapatrao Kulkarni int numa_node;
117558b0165SArd Biesheuvel unsigned int msi_domain_flags;
118558b0165SArd Biesheuvel u32 pre_its_base; /* for Socionext Synquacer */
1195c9a882eSMarc Zyngier int vlpi_redist_offset;
120cc2d3216SMarc Zyngier };
121cc2d3216SMarc Zyngier
1220dd57fedSMarc Zyngier #define is_v4(its) (!!((its)->typer & GITS_TYPER_VLPIS))
1235e516846SMarc Zyngier #define is_v4_1(its) (!!((its)->typer & GITS_TYPER_VMAPP))
124576a8342SMarc Zyngier #define device_ids(its) (FIELD_GET(GITS_TYPER_DEVBITS, (its)->typer) + 1)
1250dd57fedSMarc Zyngier
126cc2d3216SMarc Zyngier #define ITS_ITT_ALIGN SZ_256
127cc2d3216SMarc Zyngier
12832bd44dcSShanker Donthineni /* The maximum number of VPEID bits supported by VLPI commands */
129f2d83409SMarc Zyngier #define ITS_MAX_VPEID_BITS \
130f2d83409SMarc Zyngier ({ \
131f2d83409SMarc Zyngier int nvpeid = 16; \
132f2d83409SMarc Zyngier if (gic_rdists->has_rvpeid && \
133f2d83409SMarc Zyngier gic_rdists->gicd_typer2 & GICD_TYPER2_VIL) \
134f2d83409SMarc Zyngier nvpeid = 1 + (gic_rdists->gicd_typer2 & \
135f2d83409SMarc Zyngier GICD_TYPER2_VID); \
136f2d83409SMarc Zyngier \
137f2d83409SMarc Zyngier nvpeid; \
138f2d83409SMarc Zyngier })
13932bd44dcSShanker Donthineni #define ITS_MAX_VPEID (1 << (ITS_MAX_VPEID_BITS))
14032bd44dcSShanker Donthineni
1412eca0d6cSShanker Donthineni /* Convert page order to size in bytes */
1422eca0d6cSShanker Donthineni #define PAGE_ORDER_TO_SIZE(o) (PAGE_SIZE << (o))
1432eca0d6cSShanker Donthineni
144591e5becSMarc Zyngier struct event_lpi_map {
145591e5becSMarc Zyngier unsigned long *lpi_map;
146591e5becSMarc Zyngier u16 *col_map;
147591e5becSMarc Zyngier irq_hw_number_t lpi_base;
148591e5becSMarc Zyngier int nr_lpis;
14911635fa2SMarc Zyngier raw_spinlock_t vlpi_lock;
150d011e4e6SMarc Zyngier struct its_vm *vm;
151d011e4e6SMarc Zyngier struct its_vlpi_map *vlpi_maps;
152d011e4e6SMarc Zyngier int nr_vlpis;
153591e5becSMarc Zyngier };
154591e5becSMarc Zyngier
155cc2d3216SMarc Zyngier /*
156d011e4e6SMarc Zyngier * The ITS view of a device - belongs to an ITS, owns an interrupt
157d011e4e6SMarc Zyngier * translation table, and a list of interrupts. If it some of its
158d011e4e6SMarc Zyngier * LPIs are injected into a guest (GICv4), the event_map.vm field
159d011e4e6SMarc Zyngier * indicates which one.
160cc2d3216SMarc Zyngier */
161cc2d3216SMarc Zyngier struct its_device {
162cc2d3216SMarc Zyngier struct list_head entry;
163cc2d3216SMarc Zyngier struct its_node *its;
164591e5becSMarc Zyngier struct event_lpi_map event_map;
165cc2d3216SMarc Zyngier void *itt;
166cc2d3216SMarc Zyngier u32 nr_ites;
167cc2d3216SMarc Zyngier u32 device_id;
1689791ec7dSMarc Zyngier bool shared;
169cc2d3216SMarc Zyngier };
170cc2d3216SMarc Zyngier
17120b3d54eSMarc Zyngier static struct {
17220b3d54eSMarc Zyngier raw_spinlock_t lock;
17320b3d54eSMarc Zyngier struct its_device *dev;
17420b3d54eSMarc Zyngier struct its_vpe **vpes;
17520b3d54eSMarc Zyngier int next_victim;
17620b3d54eSMarc Zyngier } vpe_proxy;
17720b3d54eSMarc Zyngier
1782f13ff1dSMarc Zyngier struct cpu_lpi_count {
1792f13ff1dSMarc Zyngier atomic_t managed;
1802f13ff1dSMarc Zyngier atomic_t unmanaged;
1812f13ff1dSMarc Zyngier };
1822f13ff1dSMarc Zyngier
1832f13ff1dSMarc Zyngier static DEFINE_PER_CPU(struct cpu_lpi_count, cpu_lpi_count);
1842f13ff1dSMarc Zyngier
1851ac19ca6SMarc Zyngier static LIST_HEAD(its_nodes);
186a8db7456SSebastian Andrzej Siewior static DEFINE_RAW_SPINLOCK(its_lock);
1871ac19ca6SMarc Zyngier static struct rdists *gic_rdists;
188db40f0a7STomasz Nowicki static struct irq_domain *its_parent;
1891ac19ca6SMarc Zyngier
1903dfa576bSMarc Zyngier static unsigned long its_list_map;
1913171a47aSMarc Zyngier static u16 vmovp_seq_num;
1923171a47aSMarc Zyngier static DEFINE_RAW_SPINLOCK(vmovp_lock);
1933171a47aSMarc Zyngier
1947d75bbb4SMarc Zyngier static DEFINE_IDA(its_vpeid_ida);
1953dfa576bSMarc Zyngier
1961ac19ca6SMarc Zyngier #define gic_data_rdist() (raw_cpu_ptr(gic_rdists->rdist))
19711e37d35SMarc Zyngier #define gic_data_rdist_cpu(cpu) (per_cpu_ptr(gic_rdists->rdist, cpu))
1981ac19ca6SMarc Zyngier #define gic_data_rdist_rd_base() (gic_data_rdist()->rd_base)
199e643d803SMarc Zyngier #define gic_data_rdist_vlpi_base() (gic_data_rdist_rd_base() + SZ_128K)
2001ac19ca6SMarc Zyngier
201009384b3SMarc Zyngier /*
202009384b3SMarc Zyngier * Skip ITSs that have no vLPIs mapped, unless we're on GICv4.1, as we
203009384b3SMarc Zyngier * always have vSGIs mapped.
204009384b3SMarc Zyngier */
require_its_list_vmovp(struct its_vm * vm,struct its_node * its)205009384b3SMarc Zyngier static bool require_its_list_vmovp(struct its_vm *vm, struct its_node *its)
206009384b3SMarc Zyngier {
207009384b3SMarc Zyngier return (gic_rdists->has_rvpeid || vm->vlpi_count[its->list_nr]);
208009384b3SMarc Zyngier }
209009384b3SMarc Zyngier
rdists_support_shareable(void)210f7e84c8eSMarc Zyngier static bool rdists_support_shareable(void)
211f7e84c8eSMarc Zyngier {
212f7e84c8eSMarc Zyngier return !(gic_rdists->flags & RDIST_FLAGS_FORCE_NON_SHAREABLE);
213f7e84c8eSMarc Zyngier }
214f7e84c8eSMarc Zyngier
get_its_list(struct its_vm * vm)21584243125SZenghui Yu static u16 get_its_list(struct its_vm *vm)
21684243125SZenghui Yu {
21784243125SZenghui Yu struct its_node *its;
21884243125SZenghui Yu unsigned long its_list = 0;
21984243125SZenghui Yu
22084243125SZenghui Yu list_for_each_entry(its, &its_nodes, entry) {
2210dd57fedSMarc Zyngier if (!is_v4(its))
22284243125SZenghui Yu continue;
22384243125SZenghui Yu
224009384b3SMarc Zyngier if (require_its_list_vmovp(vm, its))
22584243125SZenghui Yu __set_bit(its->list_nr, &its_list);
22684243125SZenghui Yu }
22784243125SZenghui Yu
22884243125SZenghui Yu return (u16)its_list;
22984243125SZenghui Yu }
23084243125SZenghui Yu
its_get_event_id(struct irq_data * d)231425c09beSMarc Zyngier static inline u32 its_get_event_id(struct irq_data *d)
232425c09beSMarc Zyngier {
233425c09beSMarc Zyngier struct its_device *its_dev = irq_data_get_irq_chip_data(d);
234425c09beSMarc Zyngier return d->hwirq - its_dev->event_map.lpi_base;
235425c09beSMarc Zyngier }
236425c09beSMarc Zyngier
dev_event_to_col(struct its_device * its_dev,u32 event)237591e5becSMarc Zyngier static struct its_collection *dev_event_to_col(struct its_device *its_dev,
238591e5becSMarc Zyngier u32 event)
239591e5becSMarc Zyngier {
240591e5becSMarc Zyngier struct its_node *its = its_dev->its;
241591e5becSMarc Zyngier
242591e5becSMarc Zyngier return its->collections + its_dev->event_map.col_map[event];
243591e5becSMarc Zyngier }
244591e5becSMarc Zyngier
dev_event_to_vlpi_map(struct its_device * its_dev,u32 event)245c1d4d5cdSMarc Zyngier static struct its_vlpi_map *dev_event_to_vlpi_map(struct its_device *its_dev,
246c1d4d5cdSMarc Zyngier u32 event)
247c1d4d5cdSMarc Zyngier {
248c1d4d5cdSMarc Zyngier if (WARN_ON_ONCE(event >= its_dev->event_map.nr_lpis))
249c1d4d5cdSMarc Zyngier return NULL;
250c1d4d5cdSMarc Zyngier
251c1d4d5cdSMarc Zyngier return &its_dev->event_map.vlpi_maps[event];
252c1d4d5cdSMarc Zyngier }
253c1d4d5cdSMarc Zyngier
get_vlpi_map(struct irq_data * d)254f4a81f5aSMarc Zyngier static struct its_vlpi_map *get_vlpi_map(struct irq_data *d)
255f4a81f5aSMarc Zyngier {
256f4a81f5aSMarc Zyngier if (irqd_is_forwarded_to_vcpu(d)) {
257f4a81f5aSMarc Zyngier struct its_device *its_dev = irq_data_get_irq_chip_data(d);
258f4a81f5aSMarc Zyngier u32 event = its_get_event_id(d);
259f4a81f5aSMarc Zyngier
260f4a81f5aSMarc Zyngier return dev_event_to_vlpi_map(its_dev, event);
261f4a81f5aSMarc Zyngier }
262f4a81f5aSMarc Zyngier
263f4a81f5aSMarc Zyngier return NULL;
264f4a81f5aSMarc Zyngier }
265f4a81f5aSMarc Zyngier
vpe_to_cpuid_lock(struct its_vpe * vpe,unsigned long * flags)266f3a05921SMarc Zyngier static int vpe_to_cpuid_lock(struct its_vpe *vpe, unsigned long *flags)
267425c09beSMarc Zyngier {
268f3a05921SMarc Zyngier raw_spin_lock_irqsave(&vpe->vpe_lock, *flags);
269f3a05921SMarc Zyngier return vpe->col_idx;
270f3a05921SMarc Zyngier }
271f3a05921SMarc Zyngier
vpe_to_cpuid_unlock(struct its_vpe * vpe,unsigned long flags)272f3a05921SMarc Zyngier static void vpe_to_cpuid_unlock(struct its_vpe *vpe, unsigned long flags)
273f3a05921SMarc Zyngier {
274f3a05921SMarc Zyngier raw_spin_unlock_irqrestore(&vpe->vpe_lock, flags);
275f3a05921SMarc Zyngier }
276f3a05921SMarc Zyngier
277926846a7SMarc Zyngier static struct irq_chip its_vpe_irq_chip;
278926846a7SMarc Zyngier
irq_to_cpuid_lock(struct irq_data * d,unsigned long * flags)279f3a05921SMarc Zyngier static int irq_to_cpuid_lock(struct irq_data *d, unsigned long *flags)
280f3a05921SMarc Zyngier {
281926846a7SMarc Zyngier struct its_vpe *vpe = NULL;
282f3a05921SMarc Zyngier int cpu;
283f3a05921SMarc Zyngier
284926846a7SMarc Zyngier if (d->chip == &its_vpe_irq_chip) {
285926846a7SMarc Zyngier vpe = irq_data_get_irq_chip_data(d);
286926846a7SMarc Zyngier } else {
287926846a7SMarc Zyngier struct its_vlpi_map *map = get_vlpi_map(d);
288926846a7SMarc Zyngier if (map)
289926846a7SMarc Zyngier vpe = map->vpe;
290926846a7SMarc Zyngier }
291926846a7SMarc Zyngier
292926846a7SMarc Zyngier if (vpe) {
293926846a7SMarc Zyngier cpu = vpe_to_cpuid_lock(vpe, flags);
294f3a05921SMarc Zyngier } else {
295f3a05921SMarc Zyngier /* Physical LPIs are already locked via the irq_desc lock */
296425c09beSMarc Zyngier struct its_device *its_dev = irq_data_get_irq_chip_data(d);
297f3a05921SMarc Zyngier cpu = its_dev->event_map.col_map[its_get_event_id(d)];
298f3a05921SMarc Zyngier /* Keep GCC quiet... */
299f3a05921SMarc Zyngier *flags = 0;
300f3a05921SMarc Zyngier }
301f3a05921SMarc Zyngier
302f3a05921SMarc Zyngier return cpu;
303f3a05921SMarc Zyngier }
304f3a05921SMarc Zyngier
irq_to_cpuid_unlock(struct irq_data * d,unsigned long flags)305f3a05921SMarc Zyngier static void irq_to_cpuid_unlock(struct irq_data *d, unsigned long flags)
306f3a05921SMarc Zyngier {
307926846a7SMarc Zyngier struct its_vpe *vpe = NULL;
308425c09beSMarc Zyngier
309926846a7SMarc Zyngier if (d->chip == &its_vpe_irq_chip) {
310926846a7SMarc Zyngier vpe = irq_data_get_irq_chip_data(d);
311926846a7SMarc Zyngier } else {
312926846a7SMarc Zyngier struct its_vlpi_map *map = get_vlpi_map(d);
313f4a81f5aSMarc Zyngier if (map)
314926846a7SMarc Zyngier vpe = map->vpe;
315926846a7SMarc Zyngier }
316926846a7SMarc Zyngier
317926846a7SMarc Zyngier if (vpe)
318926846a7SMarc Zyngier vpe_to_cpuid_unlock(vpe, flags);
319425c09beSMarc Zyngier }
320425c09beSMarc Zyngier
valid_col(struct its_collection * col)32183559b47SMarc Zyngier static struct its_collection *valid_col(struct its_collection *col)
32283559b47SMarc Zyngier {
32320faba84SJoe Perches if (WARN_ON_ONCE(col->target_address & GENMASK_ULL(15, 0)))
32483559b47SMarc Zyngier return NULL;
32583559b47SMarc Zyngier
32683559b47SMarc Zyngier return col;
32783559b47SMarc Zyngier }
32883559b47SMarc Zyngier
valid_vpe(struct its_node * its,struct its_vpe * vpe)329205e065dSMarc Zyngier static struct its_vpe *valid_vpe(struct its_node *its, struct its_vpe *vpe)
330205e065dSMarc Zyngier {
331205e065dSMarc Zyngier if (valid_col(its->collections + vpe->col_idx))
332205e065dSMarc Zyngier return vpe;
333205e065dSMarc Zyngier
334205e065dSMarc Zyngier return NULL;
335205e065dSMarc Zyngier }
336205e065dSMarc Zyngier
337cc2d3216SMarc Zyngier /*
338cc2d3216SMarc Zyngier * ITS command descriptors - parameters to be encoded in a command
339cc2d3216SMarc Zyngier * block.
340cc2d3216SMarc Zyngier */
341cc2d3216SMarc Zyngier struct its_cmd_desc {
342cc2d3216SMarc Zyngier union {
343cc2d3216SMarc Zyngier struct {
344cc2d3216SMarc Zyngier struct its_device *dev;
345cc2d3216SMarc Zyngier u32 event_id;
346cc2d3216SMarc Zyngier } its_inv_cmd;
347cc2d3216SMarc Zyngier
348cc2d3216SMarc Zyngier struct {
349cc2d3216SMarc Zyngier struct its_device *dev;
350cc2d3216SMarc Zyngier u32 event_id;
3518d85dcedSMarc Zyngier } its_clear_cmd;
3528d85dcedSMarc Zyngier
3538d85dcedSMarc Zyngier struct {
3548d85dcedSMarc Zyngier struct its_device *dev;
3558d85dcedSMarc Zyngier u32 event_id;
356cc2d3216SMarc Zyngier } its_int_cmd;
357cc2d3216SMarc Zyngier
358cc2d3216SMarc Zyngier struct {
359cc2d3216SMarc Zyngier struct its_device *dev;
360cc2d3216SMarc Zyngier int valid;
361cc2d3216SMarc Zyngier } its_mapd_cmd;
362cc2d3216SMarc Zyngier
363cc2d3216SMarc Zyngier struct {
364cc2d3216SMarc Zyngier struct its_collection *col;
365cc2d3216SMarc Zyngier int valid;
366cc2d3216SMarc Zyngier } its_mapc_cmd;
367cc2d3216SMarc Zyngier
368cc2d3216SMarc Zyngier struct {
369cc2d3216SMarc Zyngier struct its_device *dev;
370cc2d3216SMarc Zyngier u32 phys_id;
371cc2d3216SMarc Zyngier u32 event_id;
3726a25ad3aSMarc Zyngier } its_mapti_cmd;
373cc2d3216SMarc Zyngier
374cc2d3216SMarc Zyngier struct {
375cc2d3216SMarc Zyngier struct its_device *dev;
376cc2d3216SMarc Zyngier struct its_collection *col;
377591e5becSMarc Zyngier u32 event_id;
378cc2d3216SMarc Zyngier } its_movi_cmd;
379cc2d3216SMarc Zyngier
380cc2d3216SMarc Zyngier struct {
381cc2d3216SMarc Zyngier struct its_device *dev;
382cc2d3216SMarc Zyngier u32 event_id;
383cc2d3216SMarc Zyngier } its_discard_cmd;
384cc2d3216SMarc Zyngier
385cc2d3216SMarc Zyngier struct {
386cc2d3216SMarc Zyngier struct its_collection *col;
387cc2d3216SMarc Zyngier } its_invall_cmd;
388d011e4e6SMarc Zyngier
389d011e4e6SMarc Zyngier struct {
390d011e4e6SMarc Zyngier struct its_vpe *vpe;
391eb78192bSMarc Zyngier } its_vinvall_cmd;
392eb78192bSMarc Zyngier
393eb78192bSMarc Zyngier struct {
394eb78192bSMarc Zyngier struct its_vpe *vpe;
395eb78192bSMarc Zyngier struct its_collection *col;
396eb78192bSMarc Zyngier bool valid;
397eb78192bSMarc Zyngier } its_vmapp_cmd;
398eb78192bSMarc Zyngier
399eb78192bSMarc Zyngier struct {
400eb78192bSMarc Zyngier struct its_vpe *vpe;
401d011e4e6SMarc Zyngier struct its_device *dev;
402d011e4e6SMarc Zyngier u32 virt_id;
403d011e4e6SMarc Zyngier u32 event_id;
404d011e4e6SMarc Zyngier bool db_enabled;
405d011e4e6SMarc Zyngier } its_vmapti_cmd;
406d011e4e6SMarc Zyngier
407d011e4e6SMarc Zyngier struct {
408d011e4e6SMarc Zyngier struct its_vpe *vpe;
409d011e4e6SMarc Zyngier struct its_device *dev;
410d011e4e6SMarc Zyngier u32 event_id;
411d011e4e6SMarc Zyngier bool db_enabled;
412d011e4e6SMarc Zyngier } its_vmovi_cmd;
4133171a47aSMarc Zyngier
4143171a47aSMarc Zyngier struct {
4153171a47aSMarc Zyngier struct its_vpe *vpe;
4163171a47aSMarc Zyngier struct its_collection *col;
4173171a47aSMarc Zyngier u16 seq_num;
4183171a47aSMarc Zyngier u16 its_list;
4193171a47aSMarc Zyngier } its_vmovp_cmd;
420d97c97baSMarc Zyngier
421d97c97baSMarc Zyngier struct {
422d97c97baSMarc Zyngier struct its_vpe *vpe;
423d97c97baSMarc Zyngier } its_invdb_cmd;
424e252cf8aSMarc Zyngier
425e252cf8aSMarc Zyngier struct {
426e252cf8aSMarc Zyngier struct its_vpe *vpe;
427e252cf8aSMarc Zyngier u8 sgi;
428e252cf8aSMarc Zyngier u8 priority;
429e252cf8aSMarc Zyngier bool enable;
430e252cf8aSMarc Zyngier bool group;
431e252cf8aSMarc Zyngier bool clear;
432e252cf8aSMarc Zyngier } its_vsgi_cmd;
433cc2d3216SMarc Zyngier };
434cc2d3216SMarc Zyngier };
435cc2d3216SMarc Zyngier
436cc2d3216SMarc Zyngier /*
437cc2d3216SMarc Zyngier * The ITS command block, which is what the ITS actually parses.
438cc2d3216SMarc Zyngier */
439cc2d3216SMarc Zyngier struct its_cmd_block {
4402bbdfcc5SBen Dooks (Codethink) union {
441cc2d3216SMarc Zyngier u64 raw_cmd[4];
4422bbdfcc5SBen Dooks (Codethink) __le64 raw_cmd_le[4];
4432bbdfcc5SBen Dooks (Codethink) };
444cc2d3216SMarc Zyngier };
445cc2d3216SMarc Zyngier
446cc2d3216SMarc Zyngier #define ITS_CMD_QUEUE_SZ SZ_64K
447cc2d3216SMarc Zyngier #define ITS_CMD_QUEUE_NR_ENTRIES (ITS_CMD_QUEUE_SZ / sizeof(struct its_cmd_block))
448cc2d3216SMarc Zyngier
44967047f90SMarc Zyngier typedef struct its_collection *(*its_cmd_builder_t)(struct its_node *,
45067047f90SMarc Zyngier struct its_cmd_block *,
451cc2d3216SMarc Zyngier struct its_cmd_desc *);
452cc2d3216SMarc Zyngier
45367047f90SMarc Zyngier typedef struct its_vpe *(*its_cmd_vbuilder_t)(struct its_node *,
45467047f90SMarc Zyngier struct its_cmd_block *,
455d011e4e6SMarc Zyngier struct its_cmd_desc *);
456d011e4e6SMarc Zyngier
4574d36f136SMarc Zyngier static void its_mask_encode(u64 *raw_cmd, u64 val, int h, int l)
4584d36f136SMarc Zyngier {
4594d36f136SMarc Zyngier u64 mask = GENMASK_ULL(h, l);
4604d36f136SMarc Zyngier *raw_cmd &= ~mask;
4614d36f136SMarc Zyngier *raw_cmd |= (val << l) & mask;
4624d36f136SMarc Zyngier }
4634d36f136SMarc Zyngier
its_encode_cmd(struct its_cmd_block * cmd,u8 cmd_nr)464cc2d3216SMarc Zyngier static void its_encode_cmd(struct its_cmd_block *cmd, u8 cmd_nr)
465cc2d3216SMarc Zyngier {
4664d36f136SMarc Zyngier its_mask_encode(&cmd->raw_cmd[0], cmd_nr, 7, 0);
467cc2d3216SMarc Zyngier }
468cc2d3216SMarc Zyngier
its_encode_devid(struct its_cmd_block * cmd,u32 devid)469cc2d3216SMarc Zyngier static void its_encode_devid(struct its_cmd_block *cmd, u32 devid)
470cc2d3216SMarc Zyngier {
4714d36f136SMarc Zyngier its_mask_encode(&cmd->raw_cmd[0], devid, 63, 32);
472cc2d3216SMarc Zyngier }
473cc2d3216SMarc Zyngier
its_encode_event_id(struct its_cmd_block * cmd,u32 id)474cc2d3216SMarc Zyngier static void its_encode_event_id(struct its_cmd_block *cmd, u32 id)
475cc2d3216SMarc Zyngier {
4764d36f136SMarc Zyngier its_mask_encode(&cmd->raw_cmd[1], id, 31, 0);
477cc2d3216SMarc Zyngier }
478cc2d3216SMarc Zyngier
its_encode_phys_id(struct its_cmd_block * cmd,u32 phys_id)479cc2d3216SMarc Zyngier static void its_encode_phys_id(struct its_cmd_block *cmd, u32 phys_id)
480cc2d3216SMarc Zyngier {
4814d36f136SMarc Zyngier its_mask_encode(&cmd->raw_cmd[1], phys_id, 63, 32);
482cc2d3216SMarc Zyngier }
483cc2d3216SMarc Zyngier
its_encode_size(struct its_cmd_block * cmd,u8 size)484cc2d3216SMarc Zyngier static void its_encode_size(struct its_cmd_block *cmd, u8 size)
485cc2d3216SMarc Zyngier {
4864d36f136SMarc Zyngier its_mask_encode(&cmd->raw_cmd[1], size, 4, 0);
487cc2d3216SMarc Zyngier }
488cc2d3216SMarc Zyngier
its_encode_itt(struct its_cmd_block * cmd,u64 itt_addr)489cc2d3216SMarc Zyngier static void its_encode_itt(struct its_cmd_block *cmd, u64 itt_addr)
490cc2d3216SMarc Zyngier {
49130ae9610SShanker Donthineni its_mask_encode(&cmd->raw_cmd[2], itt_addr >> 8, 51, 8);
492cc2d3216SMarc Zyngier }
493cc2d3216SMarc Zyngier
its_encode_valid(struct its_cmd_block * cmd,int valid)494cc2d3216SMarc Zyngier static void its_encode_valid(struct its_cmd_block *cmd, int valid)
495cc2d3216SMarc Zyngier {
4964d36f136SMarc Zyngier its_mask_encode(&cmd->raw_cmd[2], !!valid, 63, 63);
497cc2d3216SMarc Zyngier }
498cc2d3216SMarc Zyngier
its_encode_target(struct its_cmd_block * cmd,u64 target_addr)499cc2d3216SMarc Zyngier static void its_encode_target(struct its_cmd_block *cmd, u64 target_addr)
500cc2d3216SMarc Zyngier {
50130ae9610SShanker Donthineni its_mask_encode(&cmd->raw_cmd[2], target_addr >> 16, 51, 16);
502cc2d3216SMarc Zyngier }
503cc2d3216SMarc Zyngier
its_encode_collection(struct its_cmd_block * cmd,u16 col)504cc2d3216SMarc Zyngier static void its_encode_collection(struct its_cmd_block *cmd, u16 col)
505cc2d3216SMarc Zyngier {
5064d36f136SMarc Zyngier its_mask_encode(&cmd->raw_cmd[2], col, 15, 0);
507cc2d3216SMarc Zyngier }
508cc2d3216SMarc Zyngier
its_encode_vpeid(struct its_cmd_block * cmd,u16 vpeid)509d011e4e6SMarc Zyngier static void its_encode_vpeid(struct its_cmd_block *cmd, u16 vpeid)
510d011e4e6SMarc Zyngier {
511d011e4e6SMarc Zyngier its_mask_encode(&cmd->raw_cmd[1], vpeid, 47, 32);
512d011e4e6SMarc Zyngier }
513d011e4e6SMarc Zyngier
its_encode_virt_id(struct its_cmd_block * cmd,u32 virt_id)514d011e4e6SMarc Zyngier static void its_encode_virt_id(struct its_cmd_block *cmd, u32 virt_id)
515d011e4e6SMarc Zyngier {
516d011e4e6SMarc Zyngier its_mask_encode(&cmd->raw_cmd[2], virt_id, 31, 0);
517d011e4e6SMarc Zyngier }
518d011e4e6SMarc Zyngier
its_encode_db_phys_id(struct its_cmd_block * cmd,u32 db_phys_id)519d011e4e6SMarc Zyngier static void its_encode_db_phys_id(struct its_cmd_block *cmd, u32 db_phys_id)
520d011e4e6SMarc Zyngier {
521d011e4e6SMarc Zyngier its_mask_encode(&cmd->raw_cmd[2], db_phys_id, 63, 32);
522d011e4e6SMarc Zyngier }
523d011e4e6SMarc Zyngier
its_encode_db_valid(struct its_cmd_block * cmd,bool db_valid)524d011e4e6SMarc Zyngier static void its_encode_db_valid(struct its_cmd_block *cmd, bool db_valid)
525d011e4e6SMarc Zyngier {
526d011e4e6SMarc Zyngier its_mask_encode(&cmd->raw_cmd[2], db_valid, 0, 0);
527d011e4e6SMarc Zyngier }
528d011e4e6SMarc Zyngier
its_encode_seq_num(struct its_cmd_block * cmd,u16 seq_num)5293171a47aSMarc Zyngier static void its_encode_seq_num(struct its_cmd_block *cmd, u16 seq_num)
5303171a47aSMarc Zyngier {
5313171a47aSMarc Zyngier its_mask_encode(&cmd->raw_cmd[0], seq_num, 47, 32);
5323171a47aSMarc Zyngier }
5333171a47aSMarc Zyngier
its_encode_its_list(struct its_cmd_block * cmd,u16 its_list)5343171a47aSMarc Zyngier static void its_encode_its_list(struct its_cmd_block *cmd, u16 its_list)
5353171a47aSMarc Zyngier {
5363171a47aSMarc Zyngier its_mask_encode(&cmd->raw_cmd[1], its_list, 15, 0);
5373171a47aSMarc Zyngier }
5383171a47aSMarc Zyngier
its_encode_vpt_addr(struct its_cmd_block * cmd,u64 vpt_pa)539eb78192bSMarc Zyngier static void its_encode_vpt_addr(struct its_cmd_block *cmd, u64 vpt_pa)
540eb78192bSMarc Zyngier {
54130ae9610SShanker Donthineni its_mask_encode(&cmd->raw_cmd[3], vpt_pa >> 16, 51, 16);
542eb78192bSMarc Zyngier }
543eb78192bSMarc Zyngier
its_encode_vpt_size(struct its_cmd_block * cmd,u8 vpt_size)544eb78192bSMarc Zyngier static void its_encode_vpt_size(struct its_cmd_block *cmd, u8 vpt_size)
545eb78192bSMarc Zyngier {
546eb78192bSMarc Zyngier its_mask_encode(&cmd->raw_cmd[3], vpt_size, 4, 0);
547eb78192bSMarc Zyngier }
548eb78192bSMarc Zyngier
its_encode_vconf_addr(struct its_cmd_block * cmd,u64 vconf_pa)54964edfaa9SMarc Zyngier static void its_encode_vconf_addr(struct its_cmd_block *cmd, u64 vconf_pa)
55064edfaa9SMarc Zyngier {
55164edfaa9SMarc Zyngier its_mask_encode(&cmd->raw_cmd[0], vconf_pa >> 16, 51, 16);
55264edfaa9SMarc Zyngier }
55364edfaa9SMarc Zyngier
its_encode_alloc(struct its_cmd_block * cmd,bool alloc)55464edfaa9SMarc Zyngier static void its_encode_alloc(struct its_cmd_block *cmd, bool alloc)
55564edfaa9SMarc Zyngier {
55664edfaa9SMarc Zyngier its_mask_encode(&cmd->raw_cmd[0], alloc, 8, 8);
55764edfaa9SMarc Zyngier }
55864edfaa9SMarc Zyngier
its_encode_ptz(struct its_cmd_block * cmd,bool ptz)55964edfaa9SMarc Zyngier static void its_encode_ptz(struct its_cmd_block *cmd, bool ptz)
56064edfaa9SMarc Zyngier {
56164edfaa9SMarc Zyngier its_mask_encode(&cmd->raw_cmd[0], ptz, 9, 9);
56264edfaa9SMarc Zyngier }
56364edfaa9SMarc Zyngier
its_encode_vmapp_default_db(struct its_cmd_block * cmd,u32 vpe_db_lpi)56464edfaa9SMarc Zyngier static void its_encode_vmapp_default_db(struct its_cmd_block *cmd,
56564edfaa9SMarc Zyngier u32 vpe_db_lpi)
56664edfaa9SMarc Zyngier {
56764edfaa9SMarc Zyngier its_mask_encode(&cmd->raw_cmd[1], vpe_db_lpi, 31, 0);
56864edfaa9SMarc Zyngier }
56964edfaa9SMarc Zyngier
its_encode_vmovp_default_db(struct its_cmd_block * cmd,u32 vpe_db_lpi)570dd3f050aSMarc Zyngier static void its_encode_vmovp_default_db(struct its_cmd_block *cmd,
571dd3f050aSMarc Zyngier u32 vpe_db_lpi)
572dd3f050aSMarc Zyngier {
573dd3f050aSMarc Zyngier its_mask_encode(&cmd->raw_cmd[3], vpe_db_lpi, 31, 0);
574dd3f050aSMarc Zyngier }
575dd3f050aSMarc Zyngier
its_encode_db(struct its_cmd_block * cmd,bool db)576dd3f050aSMarc Zyngier static void its_encode_db(struct its_cmd_block *cmd, bool db)
577dd3f050aSMarc Zyngier {
578dd3f050aSMarc Zyngier its_mask_encode(&cmd->raw_cmd[2], db, 63, 63);
579dd3f050aSMarc Zyngier }
580dd3f050aSMarc Zyngier
its_encode_sgi_intid(struct its_cmd_block * cmd,u8 sgi)581e252cf8aSMarc Zyngier static void its_encode_sgi_intid(struct its_cmd_block *cmd, u8 sgi)
582e252cf8aSMarc Zyngier {
583e252cf8aSMarc Zyngier its_mask_encode(&cmd->raw_cmd[0], sgi, 35, 32);
584e252cf8aSMarc Zyngier }
585e252cf8aSMarc Zyngier
its_encode_sgi_priority(struct its_cmd_block * cmd,u8 prio)586e252cf8aSMarc Zyngier static void its_encode_sgi_priority(struct its_cmd_block *cmd, u8 prio)
587e252cf8aSMarc Zyngier {
588e252cf8aSMarc Zyngier its_mask_encode(&cmd->raw_cmd[0], prio >> 4, 23, 20);
589e252cf8aSMarc Zyngier }
590e252cf8aSMarc Zyngier
its_encode_sgi_group(struct its_cmd_block * cmd,bool grp)591e252cf8aSMarc Zyngier static void its_encode_sgi_group(struct its_cmd_block *cmd, bool grp)
592e252cf8aSMarc Zyngier {
593e252cf8aSMarc Zyngier its_mask_encode(&cmd->raw_cmd[0], grp, 10, 10);
594e252cf8aSMarc Zyngier }
595e252cf8aSMarc Zyngier
its_encode_sgi_clear(struct its_cmd_block * cmd,bool clr)596e252cf8aSMarc Zyngier static void its_encode_sgi_clear(struct its_cmd_block *cmd, bool clr)
597e252cf8aSMarc Zyngier {
598e252cf8aSMarc Zyngier its_mask_encode(&cmd->raw_cmd[0], clr, 9, 9);
599e252cf8aSMarc Zyngier }
600e252cf8aSMarc Zyngier
its_encode_sgi_enable(struct its_cmd_block * cmd,bool en)601e252cf8aSMarc Zyngier static void its_encode_sgi_enable(struct its_cmd_block *cmd, bool en)
602e252cf8aSMarc Zyngier {
603e252cf8aSMarc Zyngier its_mask_encode(&cmd->raw_cmd[0], en, 8, 8);
604e252cf8aSMarc Zyngier }
605e252cf8aSMarc Zyngier
its_fixup_cmd(struct its_cmd_block * cmd)606cc2d3216SMarc Zyngier static inline void its_fixup_cmd(struct its_cmd_block *cmd)
607cc2d3216SMarc Zyngier {
608cc2d3216SMarc Zyngier /* Let's fixup BE commands */
6092bbdfcc5SBen Dooks (Codethink) cmd->raw_cmd_le[0] = cpu_to_le64(cmd->raw_cmd[0]);
6102bbdfcc5SBen Dooks (Codethink) cmd->raw_cmd_le[1] = cpu_to_le64(cmd->raw_cmd[1]);
6112bbdfcc5SBen Dooks (Codethink) cmd->raw_cmd_le[2] = cpu_to_le64(cmd->raw_cmd[2]);
6122bbdfcc5SBen Dooks (Codethink) cmd->raw_cmd_le[3] = cpu_to_le64(cmd->raw_cmd[3]);
613cc2d3216SMarc Zyngier }
614cc2d3216SMarc Zyngier
its_build_mapd_cmd(struct its_node * its,struct its_cmd_block * cmd,struct its_cmd_desc * desc)61567047f90SMarc Zyngier static struct its_collection *its_build_mapd_cmd(struct its_node *its,
61667047f90SMarc Zyngier struct its_cmd_block *cmd,
617cc2d3216SMarc Zyngier struct its_cmd_desc *desc)
618cc2d3216SMarc Zyngier {
619cc2d3216SMarc Zyngier unsigned long itt_addr;
620c8481267SMarc Zyngier u8 size = ilog2(desc->its_mapd_cmd.dev->nr_ites);
621cc2d3216SMarc Zyngier
622cc2d3216SMarc Zyngier itt_addr = virt_to_phys(desc->its_mapd_cmd.dev->itt);
623cc2d3216SMarc Zyngier itt_addr = ALIGN(itt_addr, ITS_ITT_ALIGN);
624cc2d3216SMarc Zyngier
625cc2d3216SMarc Zyngier its_encode_cmd(cmd, GITS_CMD_MAPD);
626cc2d3216SMarc Zyngier its_encode_devid(cmd, desc->its_mapd_cmd.dev->device_id);
627cc2d3216SMarc Zyngier its_encode_size(cmd, size - 1);
628cc2d3216SMarc Zyngier its_encode_itt(cmd, itt_addr);
629cc2d3216SMarc Zyngier its_encode_valid(cmd, desc->its_mapd_cmd.valid);
630cc2d3216SMarc Zyngier
631cc2d3216SMarc Zyngier its_fixup_cmd(cmd);
632cc2d3216SMarc Zyngier
633591e5becSMarc Zyngier return NULL;
634cc2d3216SMarc Zyngier }
635cc2d3216SMarc Zyngier
its_build_mapc_cmd(struct its_node * its,struct its_cmd_block * cmd,struct its_cmd_desc * desc)63667047f90SMarc Zyngier static struct its_collection *its_build_mapc_cmd(struct its_node *its,
63767047f90SMarc Zyngier struct its_cmd_block *cmd,
638cc2d3216SMarc Zyngier struct its_cmd_desc *desc)
639cc2d3216SMarc Zyngier {
640cc2d3216SMarc Zyngier its_encode_cmd(cmd, GITS_CMD_MAPC);
641cc2d3216SMarc Zyngier its_encode_collection(cmd, desc->its_mapc_cmd.col->col_id);
642cc2d3216SMarc Zyngier its_encode_target(cmd, desc->its_mapc_cmd.col->target_address);
643cc2d3216SMarc Zyngier its_encode_valid(cmd, desc->its_mapc_cmd.valid);
644cc2d3216SMarc Zyngier
645cc2d3216SMarc Zyngier its_fixup_cmd(cmd);
646cc2d3216SMarc Zyngier
647cc2d3216SMarc Zyngier return desc->its_mapc_cmd.col;
648cc2d3216SMarc Zyngier }
649cc2d3216SMarc Zyngier
its_build_mapti_cmd(struct its_node * its,struct its_cmd_block * cmd,struct its_cmd_desc * desc)65067047f90SMarc Zyngier static struct its_collection *its_build_mapti_cmd(struct its_node *its,
65167047f90SMarc Zyngier struct its_cmd_block *cmd,
652cc2d3216SMarc Zyngier struct its_cmd_desc *desc)
653cc2d3216SMarc Zyngier {
654591e5becSMarc Zyngier struct its_collection *col;
655591e5becSMarc Zyngier
6566a25ad3aSMarc Zyngier col = dev_event_to_col(desc->its_mapti_cmd.dev,
6576a25ad3aSMarc Zyngier desc->its_mapti_cmd.event_id);
658591e5becSMarc Zyngier
6596a25ad3aSMarc Zyngier its_encode_cmd(cmd, GITS_CMD_MAPTI);
6606a25ad3aSMarc Zyngier its_encode_devid(cmd, desc->its_mapti_cmd.dev->device_id);
6616a25ad3aSMarc Zyngier its_encode_event_id(cmd, desc->its_mapti_cmd.event_id);
6626a25ad3aSMarc Zyngier its_encode_phys_id(cmd, desc->its_mapti_cmd.phys_id);
663591e5becSMarc Zyngier its_encode_collection(cmd, col->col_id);
664cc2d3216SMarc Zyngier
665cc2d3216SMarc Zyngier its_fixup_cmd(cmd);
666cc2d3216SMarc Zyngier
66783559b47SMarc Zyngier return valid_col(col);
668cc2d3216SMarc Zyngier }
669cc2d3216SMarc Zyngier
its_build_movi_cmd(struct its_node * its,struct its_cmd_block * cmd,struct its_cmd_desc * desc)67067047f90SMarc Zyngier static struct its_collection *its_build_movi_cmd(struct its_node *its,
67167047f90SMarc Zyngier struct its_cmd_block *cmd,
672cc2d3216SMarc Zyngier struct its_cmd_desc *desc)
673cc2d3216SMarc Zyngier {
674591e5becSMarc Zyngier struct its_collection *col;
675591e5becSMarc Zyngier
676591e5becSMarc Zyngier col = dev_event_to_col(desc->its_movi_cmd.dev,
677591e5becSMarc Zyngier desc->its_movi_cmd.event_id);
678591e5becSMarc Zyngier
679cc2d3216SMarc Zyngier its_encode_cmd(cmd, GITS_CMD_MOVI);
680cc2d3216SMarc Zyngier its_encode_devid(cmd, desc->its_movi_cmd.dev->device_id);
681591e5becSMarc Zyngier its_encode_event_id(cmd, desc->its_movi_cmd.event_id);
682cc2d3216SMarc Zyngier its_encode_collection(cmd, desc->its_movi_cmd.col->col_id);
683cc2d3216SMarc Zyngier
684cc2d3216SMarc Zyngier its_fixup_cmd(cmd);
685cc2d3216SMarc Zyngier
68683559b47SMarc Zyngier return valid_col(col);
687cc2d3216SMarc Zyngier }
688cc2d3216SMarc Zyngier
its_build_discard_cmd(struct its_node * its,struct its_cmd_block * cmd,struct its_cmd_desc * desc)68967047f90SMarc Zyngier static struct its_collection *its_build_discard_cmd(struct its_node *its,
69067047f90SMarc Zyngier struct its_cmd_block *cmd,
691cc2d3216SMarc Zyngier struct its_cmd_desc *desc)
692cc2d3216SMarc Zyngier {
693591e5becSMarc Zyngier struct its_collection *col;
694591e5becSMarc Zyngier
695591e5becSMarc Zyngier col = dev_event_to_col(desc->its_discard_cmd.dev,
696591e5becSMarc Zyngier desc->its_discard_cmd.event_id);
697591e5becSMarc Zyngier
698cc2d3216SMarc Zyngier its_encode_cmd(cmd, GITS_CMD_DISCARD);
699cc2d3216SMarc Zyngier its_encode_devid(cmd, desc->its_discard_cmd.dev->device_id);
700cc2d3216SMarc Zyngier its_encode_event_id(cmd, desc->its_discard_cmd.event_id);
701cc2d3216SMarc Zyngier
702cc2d3216SMarc Zyngier its_fixup_cmd(cmd);
703cc2d3216SMarc Zyngier
70483559b47SMarc Zyngier return valid_col(col);
705cc2d3216SMarc Zyngier }
706cc2d3216SMarc Zyngier
its_build_inv_cmd(struct its_node * its,struct its_cmd_block * cmd,struct its_cmd_desc * desc)70767047f90SMarc Zyngier static struct its_collection *its_build_inv_cmd(struct its_node *its,
70867047f90SMarc Zyngier struct its_cmd_block *cmd,
709cc2d3216SMarc Zyngier struct its_cmd_desc *desc)
710cc2d3216SMarc Zyngier {
711591e5becSMarc Zyngier struct its_collection *col;
712591e5becSMarc Zyngier
713591e5becSMarc Zyngier col = dev_event_to_col(desc->its_inv_cmd.dev,
714591e5becSMarc Zyngier desc->its_inv_cmd.event_id);
715591e5becSMarc Zyngier
716cc2d3216SMarc Zyngier its_encode_cmd(cmd, GITS_CMD_INV);
717cc2d3216SMarc Zyngier its_encode_devid(cmd, desc->its_inv_cmd.dev->device_id);
718cc2d3216SMarc Zyngier its_encode_event_id(cmd, desc->its_inv_cmd.event_id);
719cc2d3216SMarc Zyngier
720cc2d3216SMarc Zyngier its_fixup_cmd(cmd);
721cc2d3216SMarc Zyngier
72283559b47SMarc Zyngier return valid_col(col);
723cc2d3216SMarc Zyngier }
724cc2d3216SMarc Zyngier
its_build_int_cmd(struct its_node * its,struct its_cmd_block * cmd,struct its_cmd_desc * desc)72567047f90SMarc Zyngier static struct its_collection *its_build_int_cmd(struct its_node *its,
72667047f90SMarc Zyngier struct its_cmd_block *cmd,
7278d85dcedSMarc Zyngier struct its_cmd_desc *desc)
7288d85dcedSMarc Zyngier {
7298d85dcedSMarc Zyngier struct its_collection *col;
7308d85dcedSMarc Zyngier
7318d85dcedSMarc Zyngier col = dev_event_to_col(desc->its_int_cmd.dev,
7328d85dcedSMarc Zyngier desc->its_int_cmd.event_id);
7338d85dcedSMarc Zyngier
7348d85dcedSMarc Zyngier its_encode_cmd(cmd, GITS_CMD_INT);
7358d85dcedSMarc Zyngier its_encode_devid(cmd, desc->its_int_cmd.dev->device_id);
7368d85dcedSMarc Zyngier its_encode_event_id(cmd, desc->its_int_cmd.event_id);
7378d85dcedSMarc Zyngier
7388d85dcedSMarc Zyngier its_fixup_cmd(cmd);
7398d85dcedSMarc Zyngier
74083559b47SMarc Zyngier return valid_col(col);
7418d85dcedSMarc Zyngier }
7428d85dcedSMarc Zyngier
its_build_clear_cmd(struct its_node * its,struct its_cmd_block * cmd,struct its_cmd_desc * desc)74367047f90SMarc Zyngier static struct its_collection *its_build_clear_cmd(struct its_node *its,
74467047f90SMarc Zyngier struct its_cmd_block *cmd,
7458d85dcedSMarc Zyngier struct its_cmd_desc *desc)
7468d85dcedSMarc Zyngier {
7478d85dcedSMarc Zyngier struct its_collection *col;
7488d85dcedSMarc Zyngier
7498d85dcedSMarc Zyngier col = dev_event_to_col(desc->its_clear_cmd.dev,
7508d85dcedSMarc Zyngier desc->its_clear_cmd.event_id);
7518d85dcedSMarc Zyngier
7528d85dcedSMarc Zyngier its_encode_cmd(cmd, GITS_CMD_CLEAR);
7538d85dcedSMarc Zyngier its_encode_devid(cmd, desc->its_clear_cmd.dev->device_id);
7548d85dcedSMarc Zyngier its_encode_event_id(cmd, desc->its_clear_cmd.event_id);
7558d85dcedSMarc Zyngier
7568d85dcedSMarc Zyngier its_fixup_cmd(cmd);
7578d85dcedSMarc Zyngier
75883559b47SMarc Zyngier return valid_col(col);
7598d85dcedSMarc Zyngier }
7608d85dcedSMarc Zyngier
its_build_invall_cmd(struct its_node * its,struct its_cmd_block * cmd,struct its_cmd_desc * desc)76167047f90SMarc Zyngier static struct its_collection *its_build_invall_cmd(struct its_node *its,
76267047f90SMarc Zyngier struct its_cmd_block *cmd,
763cc2d3216SMarc Zyngier struct its_cmd_desc *desc)
764cc2d3216SMarc Zyngier {
765cc2d3216SMarc Zyngier its_encode_cmd(cmd, GITS_CMD_INVALL);
76610794522SZenghui Yu its_encode_collection(cmd, desc->its_invall_cmd.col->col_id);
767cc2d3216SMarc Zyngier
768cc2d3216SMarc Zyngier its_fixup_cmd(cmd);
769cc2d3216SMarc Zyngier
770b383a42cSWudi Wang return desc->its_invall_cmd.col;
771cc2d3216SMarc Zyngier }
772cc2d3216SMarc Zyngier
its_build_vinvall_cmd(struct its_node * its,struct its_cmd_block * cmd,struct its_cmd_desc * desc)77367047f90SMarc Zyngier static struct its_vpe *its_build_vinvall_cmd(struct its_node *its,
77467047f90SMarc Zyngier struct its_cmd_block *cmd,
775eb78192bSMarc Zyngier struct its_cmd_desc *desc)
776eb78192bSMarc Zyngier {
777eb78192bSMarc Zyngier its_encode_cmd(cmd, GITS_CMD_VINVALL);
778eb78192bSMarc Zyngier its_encode_vpeid(cmd, desc->its_vinvall_cmd.vpe->vpe_id);
779eb78192bSMarc Zyngier
780eb78192bSMarc Zyngier its_fixup_cmd(cmd);
781eb78192bSMarc Zyngier
782205e065dSMarc Zyngier return valid_vpe(its, desc->its_vinvall_cmd.vpe);
783eb78192bSMarc Zyngier }
784eb78192bSMarc Zyngier
its_build_vmapp_cmd(struct its_node * its,struct its_cmd_block * cmd,struct its_cmd_desc * desc)78567047f90SMarc Zyngier static struct its_vpe *its_build_vmapp_cmd(struct its_node *its,
78667047f90SMarc Zyngier struct its_cmd_block *cmd,
787eb78192bSMarc Zyngier struct its_cmd_desc *desc)
788eb78192bSMarc Zyngier {
78964edfaa9SMarc Zyngier unsigned long vpt_addr, vconf_addr;
7905c9a882eSMarc Zyngier u64 target;
79164edfaa9SMarc Zyngier bool alloc;
792eb78192bSMarc Zyngier
793eb78192bSMarc Zyngier its_encode_cmd(cmd, GITS_CMD_VMAPP);
794eb78192bSMarc Zyngier its_encode_vpeid(cmd, desc->its_vmapp_cmd.vpe->vpe_id);
795eb78192bSMarc Zyngier its_encode_valid(cmd, desc->its_vmapp_cmd.valid);
79664edfaa9SMarc Zyngier
79764edfaa9SMarc Zyngier if (!desc->its_vmapp_cmd.valid) {
79864edfaa9SMarc Zyngier if (is_v4_1(its)) {
79964edfaa9SMarc Zyngier alloc = !atomic_dec_return(&desc->its_vmapp_cmd.vpe->vmapp_count);
80064edfaa9SMarc Zyngier its_encode_alloc(cmd, alloc);
80164edfaa9SMarc Zyngier }
80264edfaa9SMarc Zyngier
80364edfaa9SMarc Zyngier goto out;
80464edfaa9SMarc Zyngier }
80564edfaa9SMarc Zyngier
80664edfaa9SMarc Zyngier vpt_addr = virt_to_phys(page_address(desc->its_vmapp_cmd.vpe->vpt_page));
80764edfaa9SMarc Zyngier target = desc->its_vmapp_cmd.col->target_address + its->vlpi_redist_offset;
80864edfaa9SMarc Zyngier
8095c9a882eSMarc Zyngier its_encode_target(cmd, target);
810eb78192bSMarc Zyngier its_encode_vpt_addr(cmd, vpt_addr);
811eb78192bSMarc Zyngier its_encode_vpt_size(cmd, LPI_NRBITS - 1);
812eb78192bSMarc Zyngier
81364edfaa9SMarc Zyngier if (!is_v4_1(its))
81464edfaa9SMarc Zyngier goto out;
81564edfaa9SMarc Zyngier
81664edfaa9SMarc Zyngier vconf_addr = virt_to_phys(page_address(desc->its_vmapp_cmd.vpe->its_vm->vprop_page));
81764edfaa9SMarc Zyngier
81864edfaa9SMarc Zyngier alloc = !atomic_fetch_inc(&desc->its_vmapp_cmd.vpe->vmapp_count);
81964edfaa9SMarc Zyngier
82064edfaa9SMarc Zyngier its_encode_alloc(cmd, alloc);
82164edfaa9SMarc Zyngier
822c21bc068SShenming Lu /*
823c21bc068SShenming Lu * GICv4.1 provides a way to get the VLPI state, which needs the vPE
824c21bc068SShenming Lu * to be unmapped first, and in this case, we may remap the vPE
825c21bc068SShenming Lu * back while the VPT is not empty. So we can't assume that the
826c21bc068SShenming Lu * VPT is empty on map. This is why we never advertise PTZ.
827c21bc068SShenming Lu */
828c21bc068SShenming Lu its_encode_ptz(cmd, false);
82964edfaa9SMarc Zyngier its_encode_vconf_addr(cmd, vconf_addr);
83064edfaa9SMarc Zyngier its_encode_vmapp_default_db(cmd, desc->its_vmapp_cmd.vpe->vpe_db_lpi);
83164edfaa9SMarc Zyngier
83264edfaa9SMarc Zyngier out:
833eb78192bSMarc Zyngier its_fixup_cmd(cmd);
834eb78192bSMarc Zyngier
835205e065dSMarc Zyngier return valid_vpe(its, desc->its_vmapp_cmd.vpe);
836eb78192bSMarc Zyngier }
837eb78192bSMarc Zyngier
its_build_vmapti_cmd(struct its_node * its,struct its_cmd_block * cmd,struct its_cmd_desc * desc)83867047f90SMarc Zyngier static struct its_vpe *its_build_vmapti_cmd(struct its_node *its,
83967047f90SMarc Zyngier struct its_cmd_block *cmd,
840d011e4e6SMarc Zyngier struct its_cmd_desc *desc)
841d011e4e6SMarc Zyngier {
842d011e4e6SMarc Zyngier u32 db;
843d011e4e6SMarc Zyngier
8443858d4dfSMarc Zyngier if (!is_v4_1(its) && desc->its_vmapti_cmd.db_enabled)
845d011e4e6SMarc Zyngier db = desc->its_vmapti_cmd.vpe->vpe_db_lpi;
846d011e4e6SMarc Zyngier else
847d011e4e6SMarc Zyngier db = 1023;
848d011e4e6SMarc Zyngier
849d011e4e6SMarc Zyngier its_encode_cmd(cmd, GITS_CMD_VMAPTI);
850d011e4e6SMarc Zyngier its_encode_devid(cmd, desc->its_vmapti_cmd.dev->device_id);
851d011e4e6SMarc Zyngier its_encode_vpeid(cmd, desc->its_vmapti_cmd.vpe->vpe_id);
852d011e4e6SMarc Zyngier its_encode_event_id(cmd, desc->its_vmapti_cmd.event_id);
853d011e4e6SMarc Zyngier its_encode_db_phys_id(cmd, db);
854d011e4e6SMarc Zyngier its_encode_virt_id(cmd, desc->its_vmapti_cmd.virt_id);
855d011e4e6SMarc Zyngier
856d011e4e6SMarc Zyngier its_fixup_cmd(cmd);
857d011e4e6SMarc Zyngier
858205e065dSMarc Zyngier return valid_vpe(its, desc->its_vmapti_cmd.vpe);
859d011e4e6SMarc Zyngier }
860d011e4e6SMarc Zyngier
its_build_vmovi_cmd(struct its_node * its,struct its_cmd_block * cmd,struct its_cmd_desc * desc)86167047f90SMarc Zyngier static struct its_vpe *its_build_vmovi_cmd(struct its_node *its,
86267047f90SMarc Zyngier struct its_cmd_block *cmd,
863d011e4e6SMarc Zyngier struct its_cmd_desc *desc)
864d011e4e6SMarc Zyngier {
865d011e4e6SMarc Zyngier u32 db;
866d011e4e6SMarc Zyngier
8673858d4dfSMarc Zyngier if (!is_v4_1(its) && desc->its_vmovi_cmd.db_enabled)
868d011e4e6SMarc Zyngier db = desc->its_vmovi_cmd.vpe->vpe_db_lpi;
869d011e4e6SMarc Zyngier else
870d011e4e6SMarc Zyngier db = 1023;
871d011e4e6SMarc Zyngier
872d011e4e6SMarc Zyngier its_encode_cmd(cmd, GITS_CMD_VMOVI);
873d011e4e6SMarc Zyngier its_encode_devid(cmd, desc->its_vmovi_cmd.dev->device_id);
874d011e4e6SMarc Zyngier its_encode_vpeid(cmd, desc->its_vmovi_cmd.vpe->vpe_id);
875d011e4e6SMarc Zyngier its_encode_event_id(cmd, desc->its_vmovi_cmd.event_id);
876d011e4e6SMarc Zyngier its_encode_db_phys_id(cmd, db);
877d011e4e6SMarc Zyngier its_encode_db_valid(cmd, true);
878d011e4e6SMarc Zyngier
879d011e4e6SMarc Zyngier its_fixup_cmd(cmd);
880d011e4e6SMarc Zyngier
881205e065dSMarc Zyngier return valid_vpe(its, desc->its_vmovi_cmd.vpe);
882d011e4e6SMarc Zyngier }
883d011e4e6SMarc Zyngier
its_build_vmovp_cmd(struct its_node * its,struct its_cmd_block * cmd,struct its_cmd_desc * desc)88467047f90SMarc Zyngier static struct its_vpe *its_build_vmovp_cmd(struct its_node *its,
88567047f90SMarc Zyngier struct its_cmd_block *cmd,
8863171a47aSMarc Zyngier struct its_cmd_desc *desc)
8873171a47aSMarc Zyngier {
8885c9a882eSMarc Zyngier u64 target;
8895c9a882eSMarc Zyngier
8905c9a882eSMarc Zyngier target = desc->its_vmovp_cmd.col->target_address + its->vlpi_redist_offset;
8913171a47aSMarc Zyngier its_encode_cmd(cmd, GITS_CMD_VMOVP);
8923171a47aSMarc Zyngier its_encode_seq_num(cmd, desc->its_vmovp_cmd.seq_num);
8933171a47aSMarc Zyngier its_encode_its_list(cmd, desc->its_vmovp_cmd.its_list);
8943171a47aSMarc Zyngier its_encode_vpeid(cmd, desc->its_vmovp_cmd.vpe->vpe_id);
8955c9a882eSMarc Zyngier its_encode_target(cmd, target);
8963171a47aSMarc Zyngier
897dd3f050aSMarc Zyngier if (is_v4_1(its)) {
898dd3f050aSMarc Zyngier its_encode_db(cmd, true);
899dd3f050aSMarc Zyngier its_encode_vmovp_default_db(cmd, desc->its_vmovp_cmd.vpe->vpe_db_lpi);
900dd3f050aSMarc Zyngier }
901dd3f050aSMarc Zyngier
9023171a47aSMarc Zyngier its_fixup_cmd(cmd);
9033171a47aSMarc Zyngier
904205e065dSMarc Zyngier return valid_vpe(its, desc->its_vmovp_cmd.vpe);
9053171a47aSMarc Zyngier }
9063171a47aSMarc Zyngier
its_build_vinv_cmd(struct its_node * its,struct its_cmd_block * cmd,struct its_cmd_desc * desc)90728614696SMarc Zyngier static struct its_vpe *its_build_vinv_cmd(struct its_node *its,
90828614696SMarc Zyngier struct its_cmd_block *cmd,
90928614696SMarc Zyngier struct its_cmd_desc *desc)
91028614696SMarc Zyngier {
91128614696SMarc Zyngier struct its_vlpi_map *map;
91228614696SMarc Zyngier
91328614696SMarc Zyngier map = dev_event_to_vlpi_map(desc->its_inv_cmd.dev,
91428614696SMarc Zyngier desc->its_inv_cmd.event_id);
91528614696SMarc Zyngier
91628614696SMarc Zyngier its_encode_cmd(cmd, GITS_CMD_INV);
91728614696SMarc Zyngier its_encode_devid(cmd, desc->its_inv_cmd.dev->device_id);
91828614696SMarc Zyngier its_encode_event_id(cmd, desc->its_inv_cmd.event_id);
91928614696SMarc Zyngier
92028614696SMarc Zyngier its_fixup_cmd(cmd);
92128614696SMarc Zyngier
92228614696SMarc Zyngier return valid_vpe(its, map->vpe);
92328614696SMarc Zyngier }
92428614696SMarc Zyngier
its_build_vint_cmd(struct its_node * its,struct its_cmd_block * cmd,struct its_cmd_desc * desc)925ed0e4aa9SMarc Zyngier static struct its_vpe *its_build_vint_cmd(struct its_node *its,
926ed0e4aa9SMarc Zyngier struct its_cmd_block *cmd,
927ed0e4aa9SMarc Zyngier struct its_cmd_desc *desc)
928ed0e4aa9SMarc Zyngier {
929ed0e4aa9SMarc Zyngier struct its_vlpi_map *map;
930ed0e4aa9SMarc Zyngier
931ed0e4aa9SMarc Zyngier map = dev_event_to_vlpi_map(desc->its_int_cmd.dev,
932ed0e4aa9SMarc Zyngier desc->its_int_cmd.event_id);
933ed0e4aa9SMarc Zyngier
934ed0e4aa9SMarc Zyngier its_encode_cmd(cmd, GITS_CMD_INT);
935ed0e4aa9SMarc Zyngier its_encode_devid(cmd, desc->its_int_cmd.dev->device_id);
936ed0e4aa9SMarc Zyngier its_encode_event_id(cmd, desc->its_int_cmd.event_id);
937ed0e4aa9SMarc Zyngier
938ed0e4aa9SMarc Zyngier its_fixup_cmd(cmd);
939ed0e4aa9SMarc Zyngier
940ed0e4aa9SMarc Zyngier return valid_vpe(its, map->vpe);
941ed0e4aa9SMarc Zyngier }
942ed0e4aa9SMarc Zyngier
its_build_vclear_cmd(struct its_node * its,struct its_cmd_block * cmd,struct its_cmd_desc * desc)943ed0e4aa9SMarc Zyngier static struct its_vpe *its_build_vclear_cmd(struct its_node *its,
944ed0e4aa9SMarc Zyngier struct its_cmd_block *cmd,
945ed0e4aa9SMarc Zyngier struct its_cmd_desc *desc)
946ed0e4aa9SMarc Zyngier {
947ed0e4aa9SMarc Zyngier struct its_vlpi_map *map;
948ed0e4aa9SMarc Zyngier
949ed0e4aa9SMarc Zyngier map = dev_event_to_vlpi_map(desc->its_clear_cmd.dev,
950ed0e4aa9SMarc Zyngier desc->its_clear_cmd.event_id);
951ed0e4aa9SMarc Zyngier
952ed0e4aa9SMarc Zyngier its_encode_cmd(cmd, GITS_CMD_CLEAR);
953ed0e4aa9SMarc Zyngier its_encode_devid(cmd, desc->its_clear_cmd.dev->device_id);
954ed0e4aa9SMarc Zyngier its_encode_event_id(cmd, desc->its_clear_cmd.event_id);
955ed0e4aa9SMarc Zyngier
956ed0e4aa9SMarc Zyngier its_fixup_cmd(cmd);
957ed0e4aa9SMarc Zyngier
958ed0e4aa9SMarc Zyngier return valid_vpe(its, map->vpe);
959ed0e4aa9SMarc Zyngier }
960ed0e4aa9SMarc Zyngier
its_build_invdb_cmd(struct its_node * its,struct its_cmd_block * cmd,struct its_cmd_desc * desc)961d97c97baSMarc Zyngier static struct its_vpe *its_build_invdb_cmd(struct its_node *its,
962d97c97baSMarc Zyngier struct its_cmd_block *cmd,
963d97c97baSMarc Zyngier struct its_cmd_desc *desc)
964d97c97baSMarc Zyngier {
965d97c97baSMarc Zyngier if (WARN_ON(!is_v4_1(its)))
966d97c97baSMarc Zyngier return NULL;
967d97c97baSMarc Zyngier
968d97c97baSMarc Zyngier its_encode_cmd(cmd, GITS_CMD_INVDB);
969d97c97baSMarc Zyngier its_encode_vpeid(cmd, desc->its_invdb_cmd.vpe->vpe_id);
970d97c97baSMarc Zyngier
971d97c97baSMarc Zyngier its_fixup_cmd(cmd);
972d97c97baSMarc Zyngier
973d97c97baSMarc Zyngier return valid_vpe(its, desc->its_invdb_cmd.vpe);
974d97c97baSMarc Zyngier }
975d97c97baSMarc Zyngier
its_build_vsgi_cmd(struct its_node * its,struct its_cmd_block * cmd,struct its_cmd_desc * desc)976e252cf8aSMarc Zyngier static struct its_vpe *its_build_vsgi_cmd(struct its_node *its,
977e252cf8aSMarc Zyngier struct its_cmd_block *cmd,
978e252cf8aSMarc Zyngier struct its_cmd_desc *desc)
979e252cf8aSMarc Zyngier {
980e252cf8aSMarc Zyngier if (WARN_ON(!is_v4_1(its)))
981e252cf8aSMarc Zyngier return NULL;
982e252cf8aSMarc Zyngier
983e252cf8aSMarc Zyngier its_encode_cmd(cmd, GITS_CMD_VSGI);
984e252cf8aSMarc Zyngier its_encode_vpeid(cmd, desc->its_vsgi_cmd.vpe->vpe_id);
985e252cf8aSMarc Zyngier its_encode_sgi_intid(cmd, desc->its_vsgi_cmd.sgi);
986e252cf8aSMarc Zyngier its_encode_sgi_priority(cmd, desc->its_vsgi_cmd.priority);
987e252cf8aSMarc Zyngier its_encode_sgi_group(cmd, desc->its_vsgi_cmd.group);
988e252cf8aSMarc Zyngier its_encode_sgi_clear(cmd, desc->its_vsgi_cmd.clear);
989e252cf8aSMarc Zyngier its_encode_sgi_enable(cmd, desc->its_vsgi_cmd.enable);
990e252cf8aSMarc Zyngier
991e252cf8aSMarc Zyngier its_fixup_cmd(cmd);
992e252cf8aSMarc Zyngier
993e252cf8aSMarc Zyngier return valid_vpe(its, desc->its_vsgi_cmd.vpe);
994e252cf8aSMarc Zyngier }
995e252cf8aSMarc Zyngier
its_cmd_ptr_to_offset(struct its_node * its,struct its_cmd_block * ptr)996cc2d3216SMarc Zyngier static u64 its_cmd_ptr_to_offset(struct its_node *its,
997cc2d3216SMarc Zyngier struct its_cmd_block *ptr)
998cc2d3216SMarc Zyngier {
999cc2d3216SMarc Zyngier return (ptr - its->cmd_base) * sizeof(*ptr);
1000cc2d3216SMarc Zyngier }
1001cc2d3216SMarc Zyngier
its_queue_full(struct its_node * its)1002cc2d3216SMarc Zyngier static int its_queue_full(struct its_node *its)
1003cc2d3216SMarc Zyngier {
1004cc2d3216SMarc Zyngier int widx;
1005cc2d3216SMarc Zyngier int ridx;
1006cc2d3216SMarc Zyngier
1007cc2d3216SMarc Zyngier widx = its->cmd_write - its->cmd_base;
1008cc2d3216SMarc Zyngier ridx = readl_relaxed(its->base + GITS_CREADR) / sizeof(struct its_cmd_block);
1009cc2d3216SMarc Zyngier
1010cc2d3216SMarc Zyngier /* This is incredibly unlikely to happen, unless the ITS locks up. */
1011cc2d3216SMarc Zyngier if (((widx + 1) % ITS_CMD_QUEUE_NR_ENTRIES) == ridx)
1012cc2d3216SMarc Zyngier return 1;
1013cc2d3216SMarc Zyngier
1014cc2d3216SMarc Zyngier return 0;
1015cc2d3216SMarc Zyngier }
1016cc2d3216SMarc Zyngier
its_allocate_entry(struct its_node * its)1017cc2d3216SMarc Zyngier static struct its_cmd_block *its_allocate_entry(struct its_node *its)
1018cc2d3216SMarc Zyngier {
1019cc2d3216SMarc Zyngier struct its_cmd_block *cmd;
1020cc2d3216SMarc Zyngier u32 count = 1000000; /* 1s! */
1021cc2d3216SMarc Zyngier
1022cc2d3216SMarc Zyngier while (its_queue_full(its)) {
1023cc2d3216SMarc Zyngier count--;
1024cc2d3216SMarc Zyngier if (!count) {
1025cc2d3216SMarc Zyngier pr_err_ratelimited("ITS queue not draining\n");
1026cc2d3216SMarc Zyngier return NULL;
1027cc2d3216SMarc Zyngier }
1028cc2d3216SMarc Zyngier cpu_relax();
1029cc2d3216SMarc Zyngier udelay(1);
1030cc2d3216SMarc Zyngier }
1031cc2d3216SMarc Zyngier
1032cc2d3216SMarc Zyngier cmd = its->cmd_write++;
1033cc2d3216SMarc Zyngier
1034cc2d3216SMarc Zyngier /* Handle queue wrapping */
1035cc2d3216SMarc Zyngier if (its->cmd_write == (its->cmd_base + ITS_CMD_QUEUE_NR_ENTRIES))
1036cc2d3216SMarc Zyngier its->cmd_write = its->cmd_base;
1037cc2d3216SMarc Zyngier
103834d677a9SMarc Zyngier /* Clear command */
103934d677a9SMarc Zyngier cmd->raw_cmd[0] = 0;
104034d677a9SMarc Zyngier cmd->raw_cmd[1] = 0;
104134d677a9SMarc Zyngier cmd->raw_cmd[2] = 0;
104234d677a9SMarc Zyngier cmd->raw_cmd[3] = 0;
104334d677a9SMarc Zyngier
1044cc2d3216SMarc Zyngier return cmd;
1045cc2d3216SMarc Zyngier }
1046cc2d3216SMarc Zyngier
its_post_commands(struct its_node * its)1047cc2d3216SMarc Zyngier static struct its_cmd_block *its_post_commands(struct its_node *its)
1048cc2d3216SMarc Zyngier {
1049cc2d3216SMarc Zyngier u64 wr = its_cmd_ptr_to_offset(its, its->cmd_write);
1050cc2d3216SMarc Zyngier
1051cc2d3216SMarc Zyngier writel_relaxed(wr, its->base + GITS_CWRITER);
1052cc2d3216SMarc Zyngier
1053cc2d3216SMarc Zyngier return its->cmd_write;
1054cc2d3216SMarc Zyngier }
1055cc2d3216SMarc Zyngier
its_flush_cmd(struct its_node * its,struct its_cmd_block * cmd)1056cc2d3216SMarc Zyngier static void its_flush_cmd(struct its_node *its, struct its_cmd_block *cmd)
1057cc2d3216SMarc Zyngier {
1058cc2d3216SMarc Zyngier /*
1059cc2d3216SMarc Zyngier * Make sure the commands written to memory are observable by
1060cc2d3216SMarc Zyngier * the ITS.
1061cc2d3216SMarc Zyngier */
1062cc2d3216SMarc Zyngier if (its->flags & ITS_FLAGS_CMDQ_NEEDS_FLUSHING)
1063328191c0SVladimir Murzin gic_flush_dcache_to_poc(cmd, sizeof(*cmd));
1064cc2d3216SMarc Zyngier else
1065cc2d3216SMarc Zyngier dsb(ishst);
1066cc2d3216SMarc Zyngier }
1067cc2d3216SMarc Zyngier
its_wait_for_range_completion(struct its_node * its,u64 prev_idx,struct its_cmd_block * to)1068a19b462fSMarc Zyngier static int its_wait_for_range_completion(struct its_node *its,
1069a050fa54SHeyi Guo u64 prev_idx,
1070cc2d3216SMarc Zyngier struct its_cmd_block *to)
1071cc2d3216SMarc Zyngier {
1072a050fa54SHeyi Guo u64 rd_idx, to_idx, linear_idx;
1073cc2d3216SMarc Zyngier u32 count = 1000000; /* 1s! */
1074cc2d3216SMarc Zyngier
1075a050fa54SHeyi Guo /* Linearize to_idx if the command set has wrapped around */
1076cc2d3216SMarc Zyngier to_idx = its_cmd_ptr_to_offset(its, to);
1077a050fa54SHeyi Guo if (to_idx < prev_idx)
1078a050fa54SHeyi Guo to_idx += ITS_CMD_QUEUE_SZ;
1079a050fa54SHeyi Guo
1080a050fa54SHeyi Guo linear_idx = prev_idx;
1081cc2d3216SMarc Zyngier
1082cc2d3216SMarc Zyngier while (1) {
1083a050fa54SHeyi Guo s64 delta;
1084a050fa54SHeyi Guo
1085cc2d3216SMarc Zyngier rd_idx = readl_relaxed(its->base + GITS_CREADR);
10869bdd8b1cSMarc Zyngier
1087a050fa54SHeyi Guo /*
1088a050fa54SHeyi Guo * Compute the read pointer progress, taking the
1089a050fa54SHeyi Guo * potential wrap-around into account.
1090a050fa54SHeyi Guo */
1091a050fa54SHeyi Guo delta = rd_idx - prev_idx;
1092a050fa54SHeyi Guo if (rd_idx < prev_idx)
1093a050fa54SHeyi Guo delta += ITS_CMD_QUEUE_SZ;
10949bdd8b1cSMarc Zyngier
1095a050fa54SHeyi Guo linear_idx += delta;
1096a050fa54SHeyi Guo if (linear_idx >= to_idx)
1097cc2d3216SMarc Zyngier break;
1098cc2d3216SMarc Zyngier
1099cc2d3216SMarc Zyngier count--;
1100cc2d3216SMarc Zyngier if (!count) {
1101a050fa54SHeyi Guo pr_err_ratelimited("ITS queue timeout (%llu %llu)\n",
1102a050fa54SHeyi Guo to_idx, linear_idx);
1103a19b462fSMarc Zyngier return -1;
1104cc2d3216SMarc Zyngier }
1105a050fa54SHeyi Guo prev_idx = rd_idx;
1106cc2d3216SMarc Zyngier cpu_relax();
1107cc2d3216SMarc Zyngier udelay(1);
1108cc2d3216SMarc Zyngier }
1109a19b462fSMarc Zyngier
1110a19b462fSMarc Zyngier return 0;
1111cc2d3216SMarc Zyngier }
1112cc2d3216SMarc Zyngier
1113e4f9094bSMarc Zyngier /* Warning, macro hell follows */
1114e4f9094bSMarc Zyngier #define BUILD_SINGLE_CMD_FUNC(name, buildtype, synctype, buildfn) \
1115e4f9094bSMarc Zyngier void name(struct its_node *its, \
1116e4f9094bSMarc Zyngier buildtype builder, \
1117e4f9094bSMarc Zyngier struct its_cmd_desc *desc) \
1118e4f9094bSMarc Zyngier { \
1119e4f9094bSMarc Zyngier struct its_cmd_block *cmd, *sync_cmd, *next_cmd; \
1120e4f9094bSMarc Zyngier synctype *sync_obj; \
1121e4f9094bSMarc Zyngier unsigned long flags; \
1122a050fa54SHeyi Guo u64 rd_idx; \
1123e4f9094bSMarc Zyngier \
1124e4f9094bSMarc Zyngier raw_spin_lock_irqsave(&its->lock, flags); \
1125e4f9094bSMarc Zyngier \
1126e4f9094bSMarc Zyngier cmd = its_allocate_entry(its); \
1127e4f9094bSMarc Zyngier if (!cmd) { /* We're soooooo screewed... */ \
1128e4f9094bSMarc Zyngier raw_spin_unlock_irqrestore(&its->lock, flags); \
1129e4f9094bSMarc Zyngier return; \
1130e4f9094bSMarc Zyngier } \
113167047f90SMarc Zyngier sync_obj = builder(its, cmd, desc); \
1132e4f9094bSMarc Zyngier its_flush_cmd(its, cmd); \
1133e4f9094bSMarc Zyngier \
1134e4f9094bSMarc Zyngier if (sync_obj) { \
1135e4f9094bSMarc Zyngier sync_cmd = its_allocate_entry(its); \
1136e4f9094bSMarc Zyngier if (!sync_cmd) \
1137e4f9094bSMarc Zyngier goto post; \
1138e4f9094bSMarc Zyngier \
113967047f90SMarc Zyngier buildfn(its, sync_cmd, sync_obj); \
1140e4f9094bSMarc Zyngier its_flush_cmd(its, sync_cmd); \
1141e4f9094bSMarc Zyngier } \
1142e4f9094bSMarc Zyngier \
1143e4f9094bSMarc Zyngier post: \
1144a050fa54SHeyi Guo rd_idx = readl_relaxed(its->base + GITS_CREADR); \
1145e4f9094bSMarc Zyngier next_cmd = its_post_commands(its); \
1146e4f9094bSMarc Zyngier raw_spin_unlock_irqrestore(&its->lock, flags); \
1147e4f9094bSMarc Zyngier \
1148a050fa54SHeyi Guo if (its_wait_for_range_completion(its, rd_idx, next_cmd)) \
1149a19b462fSMarc Zyngier pr_err_ratelimited("ITS cmd %ps failed\n", builder); \
1150e4f9094bSMarc Zyngier }
1151e4f9094bSMarc Zyngier
its_build_sync_cmd(struct its_node * its,struct its_cmd_block * sync_cmd,struct its_collection * sync_col)115267047f90SMarc Zyngier static void its_build_sync_cmd(struct its_node *its,
115367047f90SMarc Zyngier struct its_cmd_block *sync_cmd,
1154e4f9094bSMarc Zyngier struct its_collection *sync_col)
1155cc2d3216SMarc Zyngier {
1156cc2d3216SMarc Zyngier its_encode_cmd(sync_cmd, GITS_CMD_SYNC);
1157cc2d3216SMarc Zyngier its_encode_target(sync_cmd, sync_col->target_address);
1158e4f9094bSMarc Zyngier
1159cc2d3216SMarc Zyngier its_fixup_cmd(sync_cmd);
1160cc2d3216SMarc Zyngier }
1161cc2d3216SMarc Zyngier
BUILD_SINGLE_CMD_FUNC(its_send_single_command,its_cmd_builder_t,struct its_collection,its_build_sync_cmd)1162e4f9094bSMarc Zyngier static BUILD_SINGLE_CMD_FUNC(its_send_single_command, its_cmd_builder_t,
1163e4f9094bSMarc Zyngier struct its_collection, its_build_sync_cmd)
1164cc2d3216SMarc Zyngier
116567047f90SMarc Zyngier static void its_build_vsync_cmd(struct its_node *its,
116667047f90SMarc Zyngier struct its_cmd_block *sync_cmd,
1167d011e4e6SMarc Zyngier struct its_vpe *sync_vpe)
1168d011e4e6SMarc Zyngier {
1169d011e4e6SMarc Zyngier its_encode_cmd(sync_cmd, GITS_CMD_VSYNC);
1170d011e4e6SMarc Zyngier its_encode_vpeid(sync_cmd, sync_vpe->vpe_id);
1171d011e4e6SMarc Zyngier
1172d011e4e6SMarc Zyngier its_fixup_cmd(sync_cmd);
1173d011e4e6SMarc Zyngier }
1174d011e4e6SMarc Zyngier
BUILD_SINGLE_CMD_FUNC(its_send_single_vcommand,its_cmd_vbuilder_t,struct its_vpe,its_build_vsync_cmd)1175d011e4e6SMarc Zyngier static BUILD_SINGLE_CMD_FUNC(its_send_single_vcommand, its_cmd_vbuilder_t,
1176d011e4e6SMarc Zyngier struct its_vpe, its_build_vsync_cmd)
1177d011e4e6SMarc Zyngier
11788d85dcedSMarc Zyngier static void its_send_int(struct its_device *dev, u32 event_id)
11798d85dcedSMarc Zyngier {
11808d85dcedSMarc Zyngier struct its_cmd_desc desc;
11818d85dcedSMarc Zyngier
11828d85dcedSMarc Zyngier desc.its_int_cmd.dev = dev;
11838d85dcedSMarc Zyngier desc.its_int_cmd.event_id = event_id;
11848d85dcedSMarc Zyngier
11858d85dcedSMarc Zyngier its_send_single_command(dev->its, its_build_int_cmd, &desc);
11868d85dcedSMarc Zyngier }
11878d85dcedSMarc Zyngier
its_send_clear(struct its_device * dev,u32 event_id)11888d85dcedSMarc Zyngier static void its_send_clear(struct its_device *dev, u32 event_id)
11898d85dcedSMarc Zyngier {
11908d85dcedSMarc Zyngier struct its_cmd_desc desc;
11918d85dcedSMarc Zyngier
11928d85dcedSMarc Zyngier desc.its_clear_cmd.dev = dev;
11938d85dcedSMarc Zyngier desc.its_clear_cmd.event_id = event_id;
11948d85dcedSMarc Zyngier
11958d85dcedSMarc Zyngier its_send_single_command(dev->its, its_build_clear_cmd, &desc);
1196cc2d3216SMarc Zyngier }
1197cc2d3216SMarc Zyngier
its_send_inv(struct its_device * dev,u32 event_id)1198cc2d3216SMarc Zyngier static void its_send_inv(struct its_device *dev, u32 event_id)
1199cc2d3216SMarc Zyngier {
1200cc2d3216SMarc Zyngier struct its_cmd_desc desc;
1201cc2d3216SMarc Zyngier
1202cc2d3216SMarc Zyngier desc.its_inv_cmd.dev = dev;
1203cc2d3216SMarc Zyngier desc.its_inv_cmd.event_id = event_id;
1204cc2d3216SMarc Zyngier
1205cc2d3216SMarc Zyngier its_send_single_command(dev->its, its_build_inv_cmd, &desc);
1206cc2d3216SMarc Zyngier }
1207cc2d3216SMarc Zyngier
its_send_mapd(struct its_device * dev,int valid)1208cc2d3216SMarc Zyngier static void its_send_mapd(struct its_device *dev, int valid)
1209cc2d3216SMarc Zyngier {
1210cc2d3216SMarc Zyngier struct its_cmd_desc desc;
1211cc2d3216SMarc Zyngier
1212cc2d3216SMarc Zyngier desc.its_mapd_cmd.dev = dev;
1213cc2d3216SMarc Zyngier desc.its_mapd_cmd.valid = !!valid;
1214cc2d3216SMarc Zyngier
1215cc2d3216SMarc Zyngier its_send_single_command(dev->its, its_build_mapd_cmd, &desc);
1216cc2d3216SMarc Zyngier }
1217cc2d3216SMarc Zyngier
its_send_mapc(struct its_node * its,struct its_collection * col,int valid)1218cc2d3216SMarc Zyngier static void its_send_mapc(struct its_node *its, struct its_collection *col,
1219cc2d3216SMarc Zyngier int valid)
1220cc2d3216SMarc Zyngier {
1221cc2d3216SMarc Zyngier struct its_cmd_desc desc;
1222cc2d3216SMarc Zyngier
1223cc2d3216SMarc Zyngier desc.its_mapc_cmd.col = col;
1224cc2d3216SMarc Zyngier desc.its_mapc_cmd.valid = !!valid;
1225cc2d3216SMarc Zyngier
1226cc2d3216SMarc Zyngier its_send_single_command(its, its_build_mapc_cmd, &desc);
1227cc2d3216SMarc Zyngier }
1228cc2d3216SMarc Zyngier
its_send_mapti(struct its_device * dev,u32 irq_id,u32 id)12296a25ad3aSMarc Zyngier static void its_send_mapti(struct its_device *dev, u32 irq_id, u32 id)
1230cc2d3216SMarc Zyngier {
1231cc2d3216SMarc Zyngier struct its_cmd_desc desc;
1232cc2d3216SMarc Zyngier
12336a25ad3aSMarc Zyngier desc.its_mapti_cmd.dev = dev;
12346a25ad3aSMarc Zyngier desc.its_mapti_cmd.phys_id = irq_id;
12356a25ad3aSMarc Zyngier desc.its_mapti_cmd.event_id = id;
1236cc2d3216SMarc Zyngier
12376a25ad3aSMarc Zyngier its_send_single_command(dev->its, its_build_mapti_cmd, &desc);
1238cc2d3216SMarc Zyngier }
1239cc2d3216SMarc Zyngier
its_send_movi(struct its_device * dev,struct its_collection * col,u32 id)1240cc2d3216SMarc Zyngier static void its_send_movi(struct its_device *dev,
1241cc2d3216SMarc Zyngier struct its_collection *col, u32 id)
1242cc2d3216SMarc Zyngier {
1243cc2d3216SMarc Zyngier struct its_cmd_desc desc;
1244cc2d3216SMarc Zyngier
1245cc2d3216SMarc Zyngier desc.its_movi_cmd.dev = dev;
1246cc2d3216SMarc Zyngier desc.its_movi_cmd.col = col;
1247591e5becSMarc Zyngier desc.its_movi_cmd.event_id = id;
1248cc2d3216SMarc Zyngier
1249cc2d3216SMarc Zyngier its_send_single_command(dev->its, its_build_movi_cmd, &desc);
1250cc2d3216SMarc Zyngier }
1251cc2d3216SMarc Zyngier
its_send_discard(struct its_device * dev,u32 id)1252cc2d3216SMarc Zyngier static void its_send_discard(struct its_device *dev, u32 id)
1253cc2d3216SMarc Zyngier {
1254cc2d3216SMarc Zyngier struct its_cmd_desc desc;
1255cc2d3216SMarc Zyngier
1256cc2d3216SMarc Zyngier desc.its_discard_cmd.dev = dev;
1257cc2d3216SMarc Zyngier desc.its_discard_cmd.event_id = id;
1258cc2d3216SMarc Zyngier
1259cc2d3216SMarc Zyngier its_send_single_command(dev->its, its_build_discard_cmd, &desc);
1260cc2d3216SMarc Zyngier }
1261cc2d3216SMarc Zyngier
its_send_invall(struct its_node * its,struct its_collection * col)1262cc2d3216SMarc Zyngier static void its_send_invall(struct its_node *its, struct its_collection *col)
1263cc2d3216SMarc Zyngier {
1264cc2d3216SMarc Zyngier struct its_cmd_desc desc;
1265cc2d3216SMarc Zyngier
1266cc2d3216SMarc Zyngier desc.its_invall_cmd.col = col;
1267cc2d3216SMarc Zyngier
1268cc2d3216SMarc Zyngier its_send_single_command(its, its_build_invall_cmd, &desc);
1269cc2d3216SMarc Zyngier }
1270c48ed51cSMarc Zyngier
its_send_vmapti(struct its_device * dev,u32 id)1271d011e4e6SMarc Zyngier static void its_send_vmapti(struct its_device *dev, u32 id)
1272d011e4e6SMarc Zyngier {
1273c1d4d5cdSMarc Zyngier struct its_vlpi_map *map = dev_event_to_vlpi_map(dev, id);
1274d011e4e6SMarc Zyngier struct its_cmd_desc desc;
1275d011e4e6SMarc Zyngier
1276d011e4e6SMarc Zyngier desc.its_vmapti_cmd.vpe = map->vpe;
1277d011e4e6SMarc Zyngier desc.its_vmapti_cmd.dev = dev;
1278d011e4e6SMarc Zyngier desc.its_vmapti_cmd.virt_id = map->vintid;
1279d011e4e6SMarc Zyngier desc.its_vmapti_cmd.event_id = id;
1280d011e4e6SMarc Zyngier desc.its_vmapti_cmd.db_enabled = map->db_enabled;
1281d011e4e6SMarc Zyngier
1282d011e4e6SMarc Zyngier its_send_single_vcommand(dev->its, its_build_vmapti_cmd, &desc);
1283d011e4e6SMarc Zyngier }
1284d011e4e6SMarc Zyngier
its_send_vmovi(struct its_device * dev,u32 id)1285d011e4e6SMarc Zyngier static void its_send_vmovi(struct its_device *dev, u32 id)
1286d011e4e6SMarc Zyngier {
1287c1d4d5cdSMarc Zyngier struct its_vlpi_map *map = dev_event_to_vlpi_map(dev, id);
1288d011e4e6SMarc Zyngier struct its_cmd_desc desc;
1289d011e4e6SMarc Zyngier
1290d011e4e6SMarc Zyngier desc.its_vmovi_cmd.vpe = map->vpe;
1291d011e4e6SMarc Zyngier desc.its_vmovi_cmd.dev = dev;
1292d011e4e6SMarc Zyngier desc.its_vmovi_cmd.event_id = id;
1293d011e4e6SMarc Zyngier desc.its_vmovi_cmd.db_enabled = map->db_enabled;
1294d011e4e6SMarc Zyngier
1295d011e4e6SMarc Zyngier its_send_single_vcommand(dev->its, its_build_vmovi_cmd, &desc);
1296d011e4e6SMarc Zyngier }
1297d011e4e6SMarc Zyngier
its_send_vmapp(struct its_node * its,struct its_vpe * vpe,bool valid)129875fd951bSMarc Zyngier static void its_send_vmapp(struct its_node *its,
129975fd951bSMarc Zyngier struct its_vpe *vpe, bool valid)
1300eb78192bSMarc Zyngier {
1301eb78192bSMarc Zyngier struct its_cmd_desc desc;
1302eb78192bSMarc Zyngier
1303eb78192bSMarc Zyngier desc.its_vmapp_cmd.vpe = vpe;
1304eb78192bSMarc Zyngier desc.its_vmapp_cmd.valid = valid;
1305eb78192bSMarc Zyngier desc.its_vmapp_cmd.col = &its->collections[vpe->col_idx];
130675fd951bSMarc Zyngier
1307eb78192bSMarc Zyngier its_send_single_vcommand(its, its_build_vmapp_cmd, &desc);
1308eb78192bSMarc Zyngier }
1309eb78192bSMarc Zyngier
its_send_vmovp(struct its_vpe * vpe)13103171a47aSMarc Zyngier static void its_send_vmovp(struct its_vpe *vpe)
13113171a47aSMarc Zyngier {
131284243125SZenghui Yu struct its_cmd_desc desc = {};
13133171a47aSMarc Zyngier struct its_node *its;
13143171a47aSMarc Zyngier unsigned long flags;
13153171a47aSMarc Zyngier int col_id = vpe->col_idx;
13163171a47aSMarc Zyngier
13173171a47aSMarc Zyngier desc.its_vmovp_cmd.vpe = vpe;
13183171a47aSMarc Zyngier
13193171a47aSMarc Zyngier if (!its_list_map) {
13203171a47aSMarc Zyngier its = list_first_entry(&its_nodes, struct its_node, entry);
13213171a47aSMarc Zyngier desc.its_vmovp_cmd.col = &its->collections[col_id];
13223171a47aSMarc Zyngier its_send_single_vcommand(its, its_build_vmovp_cmd, &desc);
13233171a47aSMarc Zyngier return;
13243171a47aSMarc Zyngier }
13253171a47aSMarc Zyngier
13263171a47aSMarc Zyngier /*
13273171a47aSMarc Zyngier * Yet another marvel of the architecture. If using the
13283171a47aSMarc Zyngier * its_list "feature", we need to make sure that all ITSs
13293171a47aSMarc Zyngier * receive all VMOVP commands in the same order. The only way
13303171a47aSMarc Zyngier * to guarantee this is to make vmovp a serialization point.
13313171a47aSMarc Zyngier *
13323171a47aSMarc Zyngier * Wall <-- Head.
13333171a47aSMarc Zyngier */
13343171a47aSMarc Zyngier raw_spin_lock_irqsave(&vmovp_lock, flags);
13353171a47aSMarc Zyngier
13363171a47aSMarc Zyngier desc.its_vmovp_cmd.seq_num = vmovp_seq_num++;
133784243125SZenghui Yu desc.its_vmovp_cmd.its_list = get_its_list(vpe->its_vm);
13383171a47aSMarc Zyngier
13393171a47aSMarc Zyngier /* Emit VMOVPs */
13403171a47aSMarc Zyngier list_for_each_entry(its, &its_nodes, entry) {
13410dd57fedSMarc Zyngier if (!is_v4(its))
13423171a47aSMarc Zyngier continue;
13433171a47aSMarc Zyngier
1344009384b3SMarc Zyngier if (!require_its_list_vmovp(vpe->its_vm, its))
13452247e1bfSMarc Zyngier continue;
13462247e1bfSMarc Zyngier
13473171a47aSMarc Zyngier desc.its_vmovp_cmd.col = &its->collections[col_id];
13483171a47aSMarc Zyngier its_send_single_vcommand(its, its_build_vmovp_cmd, &desc);
13493171a47aSMarc Zyngier }
13503171a47aSMarc Zyngier
13513171a47aSMarc Zyngier raw_spin_unlock_irqrestore(&vmovp_lock, flags);
13523171a47aSMarc Zyngier }
13533171a47aSMarc Zyngier
its_send_vinvall(struct its_node * its,struct its_vpe * vpe)135440619a2eSMarc Zyngier static void its_send_vinvall(struct its_node *its, struct its_vpe *vpe)
1355eb78192bSMarc Zyngier {
1356eb78192bSMarc Zyngier struct its_cmd_desc desc;
1357eb78192bSMarc Zyngier
1358eb78192bSMarc Zyngier desc.its_vinvall_cmd.vpe = vpe;
1359eb78192bSMarc Zyngier its_send_single_vcommand(its, its_build_vinvall_cmd, &desc);
1360eb78192bSMarc Zyngier }
1361eb78192bSMarc Zyngier
its_send_vinv(struct its_device * dev,u32 event_id)136228614696SMarc Zyngier static void its_send_vinv(struct its_device *dev, u32 event_id)
136328614696SMarc Zyngier {
136428614696SMarc Zyngier struct its_cmd_desc desc;
136528614696SMarc Zyngier
136628614696SMarc Zyngier /*
136728614696SMarc Zyngier * There is no real VINV command. This is just a normal INV,
136828614696SMarc Zyngier * with a VSYNC instead of a SYNC.
136928614696SMarc Zyngier */
137028614696SMarc Zyngier desc.its_inv_cmd.dev = dev;
137128614696SMarc Zyngier desc.its_inv_cmd.event_id = event_id;
137228614696SMarc Zyngier
137328614696SMarc Zyngier its_send_single_vcommand(dev->its, its_build_vinv_cmd, &desc);
137428614696SMarc Zyngier }
137528614696SMarc Zyngier
its_send_vint(struct its_device * dev,u32 event_id)1376ed0e4aa9SMarc Zyngier static void its_send_vint(struct its_device *dev, u32 event_id)
1377ed0e4aa9SMarc Zyngier {
1378ed0e4aa9SMarc Zyngier struct its_cmd_desc desc;
1379ed0e4aa9SMarc Zyngier
1380ed0e4aa9SMarc Zyngier /*
1381ed0e4aa9SMarc Zyngier * There is no real VINT command. This is just a normal INT,
1382ed0e4aa9SMarc Zyngier * with a VSYNC instead of a SYNC.
1383ed0e4aa9SMarc Zyngier */
1384ed0e4aa9SMarc Zyngier desc.its_int_cmd.dev = dev;
1385ed0e4aa9SMarc Zyngier desc.its_int_cmd.event_id = event_id;
1386ed0e4aa9SMarc Zyngier
1387ed0e4aa9SMarc Zyngier its_send_single_vcommand(dev->its, its_build_vint_cmd, &desc);
1388ed0e4aa9SMarc Zyngier }
1389ed0e4aa9SMarc Zyngier
its_send_vclear(struct its_device * dev,u32 event_id)1390ed0e4aa9SMarc Zyngier static void its_send_vclear(struct its_device *dev, u32 event_id)
1391ed0e4aa9SMarc Zyngier {
1392ed0e4aa9SMarc Zyngier struct its_cmd_desc desc;
1393ed0e4aa9SMarc Zyngier
1394ed0e4aa9SMarc Zyngier /*
1395ed0e4aa9SMarc Zyngier * There is no real VCLEAR command. This is just a normal CLEAR,
1396ed0e4aa9SMarc Zyngier * with a VSYNC instead of a SYNC.
1397ed0e4aa9SMarc Zyngier */
1398ed0e4aa9SMarc Zyngier desc.its_clear_cmd.dev = dev;
1399ed0e4aa9SMarc Zyngier desc.its_clear_cmd.event_id = event_id;
1400ed0e4aa9SMarc Zyngier
1401ed0e4aa9SMarc Zyngier its_send_single_vcommand(dev->its, its_build_vclear_cmd, &desc);
1402ed0e4aa9SMarc Zyngier }
1403ed0e4aa9SMarc Zyngier
its_send_invdb(struct its_node * its,struct its_vpe * vpe)1404d97c97baSMarc Zyngier static void its_send_invdb(struct its_node *its, struct its_vpe *vpe)
1405d97c97baSMarc Zyngier {
1406d97c97baSMarc Zyngier struct its_cmd_desc desc;
1407d97c97baSMarc Zyngier
1408d97c97baSMarc Zyngier desc.its_invdb_cmd.vpe = vpe;
1409d97c97baSMarc Zyngier its_send_single_vcommand(its, its_build_invdb_cmd, &desc);
1410d97c97baSMarc Zyngier }
1411d97c97baSMarc Zyngier
1412c48ed51cSMarc Zyngier /*
1413c48ed51cSMarc Zyngier * irqchip functions - assumes MSI, mostly.
1414c48ed51cSMarc Zyngier */
lpi_write_config(struct irq_data * d,u8 clr,u8 set)1415015ec038SMarc Zyngier static void lpi_write_config(struct irq_data *d, u8 clr, u8 set)
1416c48ed51cSMarc Zyngier {
1417c1d4d5cdSMarc Zyngier struct its_vlpi_map *map = get_vlpi_map(d);
1418015ec038SMarc Zyngier irq_hw_number_t hwirq;
1419e1a2e201SMarc Zyngier void *va;
1420adcdb94eSMarc Zyngier u8 *cfg;
1421c48ed51cSMarc Zyngier
1422c1d4d5cdSMarc Zyngier if (map) {
1423c1d4d5cdSMarc Zyngier va = page_address(map->vm->vprop_page);
1424d4d7b4adSMarc Zyngier hwirq = map->vintid;
1425d4d7b4adSMarc Zyngier
1426d4d7b4adSMarc Zyngier /* Remember the updated property */
1427d4d7b4adSMarc Zyngier map->properties &= ~clr;
1428d4d7b4adSMarc Zyngier map->properties |= set | LPI_PROP_GROUP1;
1429015ec038SMarc Zyngier } else {
1430e1a2e201SMarc Zyngier va = gic_rdists->prop_table_va;
1431015ec038SMarc Zyngier hwirq = d->hwirq;
1432015ec038SMarc Zyngier }
1433adcdb94eSMarc Zyngier
1434e1a2e201SMarc Zyngier cfg = va + hwirq - 8192;
1435adcdb94eSMarc Zyngier *cfg &= ~clr;
1436015ec038SMarc Zyngier *cfg |= set | LPI_PROP_GROUP1;
1437c48ed51cSMarc Zyngier
1438c48ed51cSMarc Zyngier /*
1439c48ed51cSMarc Zyngier * Make the above write visible to the redistributors.
1440c48ed51cSMarc Zyngier * And yes, we're flushing exactly: One. Single. Byte.
1441c48ed51cSMarc Zyngier * Humpf...
1442c48ed51cSMarc Zyngier */
1443c48ed51cSMarc Zyngier if (gic_rdists->flags & RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING)
1444328191c0SVladimir Murzin gic_flush_dcache_to_poc(cfg, sizeof(*cfg));
1445c48ed51cSMarc Zyngier else
1446c48ed51cSMarc Zyngier dsb(ishst);
1447015ec038SMarc Zyngier }
1448015ec038SMarc Zyngier
wait_for_syncr(void __iomem * rdbase)14492f4f064bSMarc Zyngier static void wait_for_syncr(void __iomem *rdbase)
14502f4f064bSMarc Zyngier {
145104d80dbeSHeyi Guo while (readl_relaxed(rdbase + GICR_SYNCR) & 1)
14522f4f064bSMarc Zyngier cpu_relax();
14532f4f064bSMarc Zyngier }
14542f4f064bSMarc Zyngier
__direct_lpi_inv(struct irq_data * d,u64 val)1455926846a7SMarc Zyngier static void __direct_lpi_inv(struct irq_data *d, u64 val)
1456926846a7SMarc Zyngier {
1457926846a7SMarc Zyngier void __iomem *rdbase;
1458926846a7SMarc Zyngier unsigned long flags;
1459926846a7SMarc Zyngier int cpu;
1460926846a7SMarc Zyngier
1461926846a7SMarc Zyngier /* Target the redistributor this LPI is currently routed to */
1462926846a7SMarc Zyngier cpu = irq_to_cpuid_lock(d, &flags);
1463926846a7SMarc Zyngier raw_spin_lock(&gic_data_rdist_cpu(cpu)->rd_lock);
1464926846a7SMarc Zyngier
1465926846a7SMarc Zyngier rdbase = per_cpu_ptr(gic_rdists->rdist, cpu)->rd_base;
1466926846a7SMarc Zyngier gic_write_lpir(val, rdbase + GICR_INVLPIR);
1467926846a7SMarc Zyngier wait_for_syncr(rdbase);
1468926846a7SMarc Zyngier
1469926846a7SMarc Zyngier raw_spin_unlock(&gic_data_rdist_cpu(cpu)->rd_lock);
1470926846a7SMarc Zyngier irq_to_cpuid_unlock(d, flags);
1471926846a7SMarc Zyngier }
1472926846a7SMarc Zyngier
direct_lpi_inv(struct irq_data * d)1473425c09beSMarc Zyngier static void direct_lpi_inv(struct irq_data *d)
1474425c09beSMarc Zyngier {
1475f4a81f5aSMarc Zyngier struct its_vlpi_map *map = get_vlpi_map(d);
1476f4a81f5aSMarc Zyngier u64 val;
1477f4a81f5aSMarc Zyngier
1478f4a81f5aSMarc Zyngier if (map) {
1479f4a81f5aSMarc Zyngier struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1480f4a81f5aSMarc Zyngier
1481f4a81f5aSMarc Zyngier WARN_ON(!is_v4_1(its_dev->its));
1482f4a81f5aSMarc Zyngier
1483f4a81f5aSMarc Zyngier val = GICR_INVLPIR_V;
1484f4a81f5aSMarc Zyngier val |= FIELD_PREP(GICR_INVLPIR_VPEID, map->vpe->vpe_id);
1485f4a81f5aSMarc Zyngier val |= FIELD_PREP(GICR_INVLPIR_INTID, map->vintid);
1486f4a81f5aSMarc Zyngier } else {
1487f4a81f5aSMarc Zyngier val = d->hwirq;
1488f4a81f5aSMarc Zyngier }
1489425c09beSMarc Zyngier
1490926846a7SMarc Zyngier __direct_lpi_inv(d, val);
1491425c09beSMarc Zyngier }
1492425c09beSMarc Zyngier
lpi_update_config(struct irq_data * d,u8 clr,u8 set)1493015ec038SMarc Zyngier static void lpi_update_config(struct irq_data *d, u8 clr, u8 set)
1494015ec038SMarc Zyngier {
1495015ec038SMarc Zyngier struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1496015ec038SMarc Zyngier
1497015ec038SMarc Zyngier lpi_write_config(d, clr, set);
1498f4a81f5aSMarc Zyngier if (gic_rdists->has_direct_lpi &&
1499f4a81f5aSMarc Zyngier (is_v4_1(its_dev->its) || !irqd_is_forwarded_to_vcpu(d)))
1500425c09beSMarc Zyngier direct_lpi_inv(d);
150128614696SMarc Zyngier else if (!irqd_is_forwarded_to_vcpu(d))
1502adcdb94eSMarc Zyngier its_send_inv(its_dev, its_get_event_id(d));
150328614696SMarc Zyngier else
150428614696SMarc Zyngier its_send_vinv(its_dev, its_get_event_id(d));
1505c48ed51cSMarc Zyngier }
1506c48ed51cSMarc Zyngier
its_vlpi_set_doorbell(struct irq_data * d,bool enable)1507015ec038SMarc Zyngier static void its_vlpi_set_doorbell(struct irq_data *d, bool enable)
1508015ec038SMarc Zyngier {
1509015ec038SMarc Zyngier struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1510015ec038SMarc Zyngier u32 event = its_get_event_id(d);
1511c1d4d5cdSMarc Zyngier struct its_vlpi_map *map;
1512015ec038SMarc Zyngier
15133858d4dfSMarc Zyngier /*
15143858d4dfSMarc Zyngier * GICv4.1 does away with the per-LPI nonsense, nothing to do
15153858d4dfSMarc Zyngier * here.
15163858d4dfSMarc Zyngier */
15173858d4dfSMarc Zyngier if (is_v4_1(its_dev->its))
15183858d4dfSMarc Zyngier return;
15193858d4dfSMarc Zyngier
1520c1d4d5cdSMarc Zyngier map = dev_event_to_vlpi_map(its_dev, event);
1521c1d4d5cdSMarc Zyngier
1522c1d4d5cdSMarc Zyngier if (map->db_enabled == enable)
1523015ec038SMarc Zyngier return;
1524015ec038SMarc Zyngier
1525c1d4d5cdSMarc Zyngier map->db_enabled = enable;
1526015ec038SMarc Zyngier
1527015ec038SMarc Zyngier /*
1528015ec038SMarc Zyngier * More fun with the architecture:
1529015ec038SMarc Zyngier *
1530015ec038SMarc Zyngier * Ideally, we'd issue a VMAPTI to set the doorbell to its LPI
1531015ec038SMarc Zyngier * value or to 1023, depending on the enable bit. But that
1532a359f757SIngo Molnar * would be issuing a mapping for an /existing/ DevID+EventID
1533015ec038SMarc Zyngier * pair, which is UNPREDICTABLE. Instead, let's issue a VMOVI
1534015ec038SMarc Zyngier * to the /same/ vPE, using this opportunity to adjust the
1535015ec038SMarc Zyngier * doorbell. Mouahahahaha. We loves it, Precious.
1536015ec038SMarc Zyngier */
1537015ec038SMarc Zyngier its_send_vmovi(its_dev, event);
1538c48ed51cSMarc Zyngier }
1539c48ed51cSMarc Zyngier
its_mask_irq(struct irq_data * d)1540c48ed51cSMarc Zyngier static void its_mask_irq(struct irq_data *d)
1541c48ed51cSMarc Zyngier {
1542015ec038SMarc Zyngier if (irqd_is_forwarded_to_vcpu(d))
1543015ec038SMarc Zyngier its_vlpi_set_doorbell(d, false);
1544015ec038SMarc Zyngier
1545adcdb94eSMarc Zyngier lpi_update_config(d, LPI_PROP_ENABLED, 0);
1546c48ed51cSMarc Zyngier }
1547c48ed51cSMarc Zyngier
its_unmask_irq(struct irq_data * d)1548c48ed51cSMarc Zyngier static void its_unmask_irq(struct irq_data *d)
1549c48ed51cSMarc Zyngier {
1550015ec038SMarc Zyngier if (irqd_is_forwarded_to_vcpu(d))
1551015ec038SMarc Zyngier its_vlpi_set_doorbell(d, true);
1552015ec038SMarc Zyngier
1553adcdb94eSMarc Zyngier lpi_update_config(d, 0, LPI_PROP_ENABLED);
1554c48ed51cSMarc Zyngier }
1555c48ed51cSMarc Zyngier
its_read_lpi_count(struct irq_data * d,int cpu)15562f13ff1dSMarc Zyngier static __maybe_unused u32 its_read_lpi_count(struct irq_data *d, int cpu)
15572f13ff1dSMarc Zyngier {
15582f13ff1dSMarc Zyngier if (irqd_affinity_is_managed(d))
15592f13ff1dSMarc Zyngier return atomic_read(&per_cpu_ptr(&cpu_lpi_count, cpu)->managed);
15602f13ff1dSMarc Zyngier
15612f13ff1dSMarc Zyngier return atomic_read(&per_cpu_ptr(&cpu_lpi_count, cpu)->unmanaged);
15622f13ff1dSMarc Zyngier }
15632f13ff1dSMarc Zyngier
its_inc_lpi_count(struct irq_data * d,int cpu)15642f13ff1dSMarc Zyngier static void its_inc_lpi_count(struct irq_data *d, int cpu)
15652f13ff1dSMarc Zyngier {
15662f13ff1dSMarc Zyngier if (irqd_affinity_is_managed(d))
15672f13ff1dSMarc Zyngier atomic_inc(&per_cpu_ptr(&cpu_lpi_count, cpu)->managed);
15682f13ff1dSMarc Zyngier else
15692f13ff1dSMarc Zyngier atomic_inc(&per_cpu_ptr(&cpu_lpi_count, cpu)->unmanaged);
15702f13ff1dSMarc Zyngier }
15712f13ff1dSMarc Zyngier
its_dec_lpi_count(struct irq_data * d,int cpu)15722f13ff1dSMarc Zyngier static void its_dec_lpi_count(struct irq_data *d, int cpu)
15732f13ff1dSMarc Zyngier {
15742f13ff1dSMarc Zyngier if (irqd_affinity_is_managed(d))
15752f13ff1dSMarc Zyngier atomic_dec(&per_cpu_ptr(&cpu_lpi_count, cpu)->managed);
15762f13ff1dSMarc Zyngier else
15772f13ff1dSMarc Zyngier atomic_dec(&per_cpu_ptr(&cpu_lpi_count, cpu)->unmanaged);
15782f13ff1dSMarc Zyngier }
15792f13ff1dSMarc Zyngier
cpumask_pick_least_loaded(struct irq_data * d,const struct cpumask * cpu_mask)1580c5d6082dSMarc Zyngier static unsigned int cpumask_pick_least_loaded(struct irq_data *d,
1581c5d6082dSMarc Zyngier const struct cpumask *cpu_mask)
1582c5d6082dSMarc Zyngier {
1583c5d6082dSMarc Zyngier unsigned int cpu = nr_cpu_ids, tmp;
1584c5d6082dSMarc Zyngier int count = S32_MAX;
1585c5d6082dSMarc Zyngier
1586c5d6082dSMarc Zyngier for_each_cpu(tmp, cpu_mask) {
1587c5d6082dSMarc Zyngier int this_count = its_read_lpi_count(d, tmp);
1588c5d6082dSMarc Zyngier if (this_count < count) {
1589c5d6082dSMarc Zyngier cpu = tmp;
1590c5d6082dSMarc Zyngier count = this_count;
1591c5d6082dSMarc Zyngier }
1592c5d6082dSMarc Zyngier }
1593c5d6082dSMarc Zyngier
1594c5d6082dSMarc Zyngier return cpu;
1595c5d6082dSMarc Zyngier }
1596c5d6082dSMarc Zyngier
1597c5d6082dSMarc Zyngier /*
1598c5d6082dSMarc Zyngier * As suggested by Thomas Gleixner in:
1599c5d6082dSMarc Zyngier * https://lore.kernel.org/r/87h80q2aoc.fsf@nanos.tec.linutronix.de
1600c5d6082dSMarc Zyngier */
its_select_cpu(struct irq_data * d,const struct cpumask * aff_mask)1601c5d6082dSMarc Zyngier static int its_select_cpu(struct irq_data *d,
1602c5d6082dSMarc Zyngier const struct cpumask *aff_mask)
1603c5d6082dSMarc Zyngier {
1604c5d6082dSMarc Zyngier struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1605f55a9b59SPierre Gondois static DEFINE_RAW_SPINLOCK(tmpmask_lock);
1606f55a9b59SPierre Gondois static struct cpumask __tmpmask;
1607f55a9b59SPierre Gondois struct cpumask *tmpmask;
1608f55a9b59SPierre Gondois unsigned long flags;
1609c5d6082dSMarc Zyngier int cpu, node;
1610c5d6082dSMarc Zyngier node = its_dev->its->numa_node;
1611f55a9b59SPierre Gondois tmpmask = &__tmpmask;
1612f55a9b59SPierre Gondois
1613f55a9b59SPierre Gondois raw_spin_lock_irqsave(&tmpmask_lock, flags);
1614c5d6082dSMarc Zyngier
1615c5d6082dSMarc Zyngier if (!irqd_affinity_is_managed(d)) {
1616c5d6082dSMarc Zyngier /* First try the NUMA node */
1617c5d6082dSMarc Zyngier if (node != NUMA_NO_NODE) {
1618c5d6082dSMarc Zyngier /*
1619c5d6082dSMarc Zyngier * Try the intersection of the affinity mask and the
1620c5d6082dSMarc Zyngier * node mask (and the online mask, just to be safe).
1621c5d6082dSMarc Zyngier */
1622c5d6082dSMarc Zyngier cpumask_and(tmpmask, cpumask_of_node(node), aff_mask);
1623c5d6082dSMarc Zyngier cpumask_and(tmpmask, tmpmask, cpu_online_mask);
1624c5d6082dSMarc Zyngier
1625c5d6082dSMarc Zyngier /*
1626c5d6082dSMarc Zyngier * Ideally, we would check if the mask is empty, and
1627c5d6082dSMarc Zyngier * try again on the full node here.
1628c5d6082dSMarc Zyngier *
1629c5d6082dSMarc Zyngier * But it turns out that the way ACPI describes the
1630c5d6082dSMarc Zyngier * affinity for ITSs only deals about memory, and
1631c5d6082dSMarc Zyngier * not target CPUs, so it cannot describe a single
1632c5d6082dSMarc Zyngier * ITS placed next to two NUMA nodes.
1633c5d6082dSMarc Zyngier *
1634c5d6082dSMarc Zyngier * Instead, just fallback on the online mask. This
1635c5d6082dSMarc Zyngier * diverges from Thomas' suggestion above.
1636c5d6082dSMarc Zyngier */
1637c5d6082dSMarc Zyngier cpu = cpumask_pick_least_loaded(d, tmpmask);
1638c5d6082dSMarc Zyngier if (cpu < nr_cpu_ids)
1639c5d6082dSMarc Zyngier goto out;
1640c5d6082dSMarc Zyngier
1641c5d6082dSMarc Zyngier /* If we can't cross sockets, give up */
1642c5d6082dSMarc Zyngier if ((its_dev->its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144))
1643c5d6082dSMarc Zyngier goto out;
1644c5d6082dSMarc Zyngier
1645c5d6082dSMarc Zyngier /* If the above failed, expand the search */
1646c5d6082dSMarc Zyngier }
1647c5d6082dSMarc Zyngier
1648c5d6082dSMarc Zyngier /* Try the intersection of the affinity and online masks */
1649c5d6082dSMarc Zyngier cpumask_and(tmpmask, aff_mask, cpu_online_mask);
1650c5d6082dSMarc Zyngier
1651c5d6082dSMarc Zyngier /* If that doesn't fly, the online mask is the last resort */
1652c5d6082dSMarc Zyngier if (cpumask_empty(tmpmask))
1653c5d6082dSMarc Zyngier cpumask_copy(tmpmask, cpu_online_mask);
1654c5d6082dSMarc Zyngier
1655c5d6082dSMarc Zyngier cpu = cpumask_pick_least_loaded(d, tmpmask);
1656c5d6082dSMarc Zyngier } else {
16573f893a59SMarc Zyngier cpumask_copy(tmpmask, aff_mask);
1658c5d6082dSMarc Zyngier
1659c5d6082dSMarc Zyngier /* If we cannot cross sockets, limit the search to that node */
1660c5d6082dSMarc Zyngier if ((its_dev->its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144) &&
1661c5d6082dSMarc Zyngier node != NUMA_NO_NODE)
1662c5d6082dSMarc Zyngier cpumask_and(tmpmask, tmpmask, cpumask_of_node(node));
1663c5d6082dSMarc Zyngier
1664c5d6082dSMarc Zyngier cpu = cpumask_pick_least_loaded(d, tmpmask);
1665c5d6082dSMarc Zyngier }
1666c5d6082dSMarc Zyngier out:
1667f55a9b59SPierre Gondois raw_spin_unlock_irqrestore(&tmpmask_lock, flags);
1668c5d6082dSMarc Zyngier
1669c5d6082dSMarc Zyngier pr_debug("IRQ%d -> %*pbl CPU%d\n", d->irq, cpumask_pr_args(aff_mask), cpu);
1670c5d6082dSMarc Zyngier return cpu;
1671c5d6082dSMarc Zyngier }
1672c5d6082dSMarc Zyngier
its_set_affinity(struct irq_data * d,const struct cpumask * mask_val,bool force)1673c48ed51cSMarc Zyngier static int its_set_affinity(struct irq_data *d, const struct cpumask *mask_val,
1674c48ed51cSMarc Zyngier bool force)
1675c48ed51cSMarc Zyngier {
1676c48ed51cSMarc Zyngier struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1677c48ed51cSMarc Zyngier struct its_collection *target_col;
1678c48ed51cSMarc Zyngier u32 id = its_get_event_id(d);
1679c5d6082dSMarc Zyngier int cpu, prev_cpu;
1680c48ed51cSMarc Zyngier
1681015ec038SMarc Zyngier /* A forwarded interrupt should use irq_set_vcpu_affinity */
1682015ec038SMarc Zyngier if (irqd_is_forwarded_to_vcpu(d))
1683015ec038SMarc Zyngier return -EINVAL;
1684015ec038SMarc Zyngier
16852f13ff1dSMarc Zyngier prev_cpu = its_dev->event_map.col_map[id];
16862f13ff1dSMarc Zyngier its_dec_lpi_count(d, prev_cpu);
16872f13ff1dSMarc Zyngier
1688c5d6082dSMarc Zyngier if (!force)
1689c5d6082dSMarc Zyngier cpu = its_select_cpu(d, mask_val);
1690c5d6082dSMarc Zyngier else
1691c5d6082dSMarc Zyngier cpu = cpumask_pick_least_loaded(d, mask_val);
1692fbf8f40eSGanapatrao Kulkarni
1693c5d6082dSMarc Zyngier if (cpu < 0 || cpu >= nr_cpu_ids)
16942f13ff1dSMarc Zyngier goto err;
1695c48ed51cSMarc Zyngier
16968b8d94a7SMaJun /* don't set the affinity when the target cpu is same as current one */
16972f13ff1dSMarc Zyngier if (cpu != prev_cpu) {
1698c48ed51cSMarc Zyngier target_col = &its_dev->its->collections[cpu];
1699c48ed51cSMarc Zyngier its_send_movi(its_dev, target_col, id);
1700591e5becSMarc Zyngier its_dev->event_map.col_map[id] = cpu;
17010d224d35SMarc Zyngier irq_data_update_effective_affinity(d, cpumask_of(cpu));
17028b8d94a7SMaJun }
1703c48ed51cSMarc Zyngier
17042f13ff1dSMarc Zyngier its_inc_lpi_count(d, cpu);
17052f13ff1dSMarc Zyngier
1706c48ed51cSMarc Zyngier return IRQ_SET_MASK_OK_DONE;
17072f13ff1dSMarc Zyngier
17082f13ff1dSMarc Zyngier err:
17092f13ff1dSMarc Zyngier its_inc_lpi_count(d, prev_cpu);
17102f13ff1dSMarc Zyngier return -EINVAL;
1711c48ed51cSMarc Zyngier }
1712c48ed51cSMarc Zyngier
its_irq_get_msi_base(struct its_device * its_dev)1713558b0165SArd Biesheuvel static u64 its_irq_get_msi_base(struct its_device *its_dev)
1714558b0165SArd Biesheuvel {
1715558b0165SArd Biesheuvel struct its_node *its = its_dev->its;
1716558b0165SArd Biesheuvel
1717558b0165SArd Biesheuvel return its->phys_base + GITS_TRANSLATER;
1718558b0165SArd Biesheuvel }
1719558b0165SArd Biesheuvel
its_irq_compose_msi_msg(struct irq_data * d,struct msi_msg * msg)1720b48ac83dSMarc Zyngier static void its_irq_compose_msi_msg(struct irq_data *d, struct msi_msg *msg)
1721b48ac83dSMarc Zyngier {
1722b48ac83dSMarc Zyngier struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1723b48ac83dSMarc Zyngier struct its_node *its;
1724b48ac83dSMarc Zyngier u64 addr;
1725b48ac83dSMarc Zyngier
1726b48ac83dSMarc Zyngier its = its_dev->its;
1727558b0165SArd Biesheuvel addr = its->get_msi_base(its_dev);
1728b48ac83dSMarc Zyngier
1729b11283ebSVladimir Murzin msg->address_lo = lower_32_bits(addr);
1730b11283ebSVladimir Murzin msg->address_hi = upper_32_bits(addr);
1731b48ac83dSMarc Zyngier msg->data = its_get_event_id(d);
173244bb7e24SRobin Murphy
173335ae7df2SJulien Grall iommu_dma_compose_msi_msg(irq_data_get_msi_desc(d), msg);
1734b48ac83dSMarc Zyngier }
1735b48ac83dSMarc Zyngier
its_irq_set_irqchip_state(struct irq_data * d,enum irqchip_irq_state which,bool state)17368d85dcedSMarc Zyngier static int its_irq_set_irqchip_state(struct irq_data *d,
17378d85dcedSMarc Zyngier enum irqchip_irq_state which,
17388d85dcedSMarc Zyngier bool state)
17398d85dcedSMarc Zyngier {
17408d85dcedSMarc Zyngier struct its_device *its_dev = irq_data_get_irq_chip_data(d);
17418d85dcedSMarc Zyngier u32 event = its_get_event_id(d);
17428d85dcedSMarc Zyngier
17438d85dcedSMarc Zyngier if (which != IRQCHIP_STATE_PENDING)
17448d85dcedSMarc Zyngier return -EINVAL;
17458d85dcedSMarc Zyngier
1746ed0e4aa9SMarc Zyngier if (irqd_is_forwarded_to_vcpu(d)) {
1747ed0e4aa9SMarc Zyngier if (state)
1748ed0e4aa9SMarc Zyngier its_send_vint(its_dev, event);
1749ed0e4aa9SMarc Zyngier else
1750ed0e4aa9SMarc Zyngier its_send_vclear(its_dev, event);
1751ed0e4aa9SMarc Zyngier } else {
17528d85dcedSMarc Zyngier if (state)
17538d85dcedSMarc Zyngier its_send_int(its_dev, event);
17548d85dcedSMarc Zyngier else
17558d85dcedSMarc Zyngier its_send_clear(its_dev, event);
1756ed0e4aa9SMarc Zyngier }
17578d85dcedSMarc Zyngier
17588d85dcedSMarc Zyngier return 0;
17598d85dcedSMarc Zyngier }
17608d85dcedSMarc Zyngier
its_irq_retrigger(struct irq_data * d)17615f774f5eSMarc Zyngier static int its_irq_retrigger(struct irq_data *d)
17625f774f5eSMarc Zyngier {
17635f774f5eSMarc Zyngier return !its_irq_set_irqchip_state(d, IRQCHIP_STATE_PENDING, true);
17645f774f5eSMarc Zyngier }
17655f774f5eSMarc Zyngier
1766009384b3SMarc Zyngier /*
1767009384b3SMarc Zyngier * Two favourable cases:
1768009384b3SMarc Zyngier *
1769009384b3SMarc Zyngier * (a) Either we have a GICv4.1, and all vPEs have to be mapped at all times
1770009384b3SMarc Zyngier * for vSGI delivery
1771009384b3SMarc Zyngier *
1772009384b3SMarc Zyngier * (b) Or the ITSs do not use a list map, meaning that VMOVP is cheap enough
1773009384b3SMarc Zyngier * and we're better off mapping all VPEs always
1774009384b3SMarc Zyngier *
1775009384b3SMarc Zyngier * If neither (a) nor (b) is true, then we map vPEs on demand.
1776009384b3SMarc Zyngier *
1777009384b3SMarc Zyngier */
gic_requires_eager_mapping(void)1778009384b3SMarc Zyngier static bool gic_requires_eager_mapping(void)
1779009384b3SMarc Zyngier {
1780009384b3SMarc Zyngier if (!its_list_map || gic_rdists->has_rvpeid)
1781009384b3SMarc Zyngier return true;
1782009384b3SMarc Zyngier
1783009384b3SMarc Zyngier return false;
1784009384b3SMarc Zyngier }
1785009384b3SMarc Zyngier
its_map_vm(struct its_node * its,struct its_vm * vm)17862247e1bfSMarc Zyngier static void its_map_vm(struct its_node *its, struct its_vm *vm)
17872247e1bfSMarc Zyngier {
17882247e1bfSMarc Zyngier unsigned long flags;
17892247e1bfSMarc Zyngier
1790009384b3SMarc Zyngier if (gic_requires_eager_mapping())
17912247e1bfSMarc Zyngier return;
17922247e1bfSMarc Zyngier
17932247e1bfSMarc Zyngier raw_spin_lock_irqsave(&vmovp_lock, flags);
17942247e1bfSMarc Zyngier
17952247e1bfSMarc Zyngier /*
17962247e1bfSMarc Zyngier * If the VM wasn't mapped yet, iterate over the vpes and get
17972247e1bfSMarc Zyngier * them mapped now.
17982247e1bfSMarc Zyngier */
17992247e1bfSMarc Zyngier vm->vlpi_count[its->list_nr]++;
18002247e1bfSMarc Zyngier
18012247e1bfSMarc Zyngier if (vm->vlpi_count[its->list_nr] == 1) {
18022247e1bfSMarc Zyngier int i;
18032247e1bfSMarc Zyngier
18042247e1bfSMarc Zyngier for (i = 0; i < vm->nr_vpes; i++) {
18052247e1bfSMarc Zyngier struct its_vpe *vpe = vm->vpes[i];
180644c4c25eSMarc Zyngier struct irq_data *d = irq_get_irq_data(vpe->irq);
18072247e1bfSMarc Zyngier
18082247e1bfSMarc Zyngier /* Map the VPE to the first possible CPU */
18092247e1bfSMarc Zyngier vpe->col_idx = cpumask_first(cpu_online_mask);
18102247e1bfSMarc Zyngier its_send_vmapp(its, vpe, true);
18112247e1bfSMarc Zyngier its_send_vinvall(its, vpe);
181244c4c25eSMarc Zyngier irq_data_update_effective_affinity(d, cpumask_of(vpe->col_idx));
18132247e1bfSMarc Zyngier }
18142247e1bfSMarc Zyngier }
18152247e1bfSMarc Zyngier
18162247e1bfSMarc Zyngier raw_spin_unlock_irqrestore(&vmovp_lock, flags);
18172247e1bfSMarc Zyngier }
18182247e1bfSMarc Zyngier
its_unmap_vm(struct its_node * its,struct its_vm * vm)18192247e1bfSMarc Zyngier static void its_unmap_vm(struct its_node *its, struct its_vm *vm)
18202247e1bfSMarc Zyngier {
18212247e1bfSMarc Zyngier unsigned long flags;
18222247e1bfSMarc Zyngier
18232247e1bfSMarc Zyngier /* Not using the ITS list? Everything is always mapped. */
1824009384b3SMarc Zyngier if (gic_requires_eager_mapping())
18252247e1bfSMarc Zyngier return;
18262247e1bfSMarc Zyngier
18272247e1bfSMarc Zyngier raw_spin_lock_irqsave(&vmovp_lock, flags);
18282247e1bfSMarc Zyngier
18292247e1bfSMarc Zyngier if (!--vm->vlpi_count[its->list_nr]) {
18302247e1bfSMarc Zyngier int i;
18312247e1bfSMarc Zyngier
18322247e1bfSMarc Zyngier for (i = 0; i < vm->nr_vpes; i++)
18332247e1bfSMarc Zyngier its_send_vmapp(its, vm->vpes[i], false);
18342247e1bfSMarc Zyngier }
18352247e1bfSMarc Zyngier
18362247e1bfSMarc Zyngier raw_spin_unlock_irqrestore(&vmovp_lock, flags);
18372247e1bfSMarc Zyngier }
18382247e1bfSMarc Zyngier
its_vlpi_map(struct irq_data * d,struct its_cmd_info * info)1839d011e4e6SMarc Zyngier static int its_vlpi_map(struct irq_data *d, struct its_cmd_info *info)
1840d011e4e6SMarc Zyngier {
1841d011e4e6SMarc Zyngier struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1842d011e4e6SMarc Zyngier u32 event = its_get_event_id(d);
1843d011e4e6SMarc Zyngier
1844d011e4e6SMarc Zyngier if (!info->map)
1845d011e4e6SMarc Zyngier return -EINVAL;
1846d011e4e6SMarc Zyngier
1847d011e4e6SMarc Zyngier if (!its_dev->event_map.vm) {
1848d011e4e6SMarc Zyngier struct its_vlpi_map *maps;
1849d011e4e6SMarc Zyngier
18506396bb22SKees Cook maps = kcalloc(its_dev->event_map.nr_lpis, sizeof(*maps),
185111635fa2SMarc Zyngier GFP_ATOMIC);
18525c0fb9cbSHagar Hemdan if (!maps)
18535c0fb9cbSHagar Hemdan return -ENOMEM;
1854d011e4e6SMarc Zyngier
1855d011e4e6SMarc Zyngier its_dev->event_map.vm = info->map->vm;
1856d011e4e6SMarc Zyngier its_dev->event_map.vlpi_maps = maps;
1857d011e4e6SMarc Zyngier } else if (its_dev->event_map.vm != info->map->vm) {
18585c0fb9cbSHagar Hemdan return -EINVAL;
1859d011e4e6SMarc Zyngier }
1860d011e4e6SMarc Zyngier
1861d011e4e6SMarc Zyngier /* Get our private copy of the mapping information */
1862d011e4e6SMarc Zyngier its_dev->event_map.vlpi_maps[event] = *info->map;
1863d011e4e6SMarc Zyngier
1864d011e4e6SMarc Zyngier if (irqd_is_forwarded_to_vcpu(d)) {
1865d011e4e6SMarc Zyngier /* Already mapped, move it around */
1866d011e4e6SMarc Zyngier its_send_vmovi(its_dev, event);
1867d011e4e6SMarc Zyngier } else {
18682247e1bfSMarc Zyngier /* Ensure all the VPEs are mapped on this ITS */
18692247e1bfSMarc Zyngier its_map_vm(its_dev->its, info->map->vm);
18702247e1bfSMarc Zyngier
1871d4d7b4adSMarc Zyngier /*
1872d4d7b4adSMarc Zyngier * Flag the interrupt as forwarded so that we can
1873d4d7b4adSMarc Zyngier * start poking the virtual property table.
1874d4d7b4adSMarc Zyngier */
1875d4d7b4adSMarc Zyngier irqd_set_forwarded_to_vcpu(d);
1876d4d7b4adSMarc Zyngier
1877d4d7b4adSMarc Zyngier /* Write out the property to the prop table */
1878d4d7b4adSMarc Zyngier lpi_write_config(d, 0xff, info->map->properties);
1879d4d7b4adSMarc Zyngier
1880d011e4e6SMarc Zyngier /* Drop the physical mapping */
1881d011e4e6SMarc Zyngier its_send_discard(its_dev, event);
1882d011e4e6SMarc Zyngier
1883d011e4e6SMarc Zyngier /* and install the virtual one */
1884d011e4e6SMarc Zyngier its_send_vmapti(its_dev, event);
1885d011e4e6SMarc Zyngier
1886d011e4e6SMarc Zyngier /* Increment the number of VLPIs */
1887d011e4e6SMarc Zyngier its_dev->event_map.nr_vlpis++;
1888d011e4e6SMarc Zyngier }
1889d011e4e6SMarc Zyngier
18905c0fb9cbSHagar Hemdan return 0;
1891d011e4e6SMarc Zyngier }
1892d011e4e6SMarc Zyngier
its_vlpi_get(struct irq_data * d,struct its_cmd_info * info)1893d011e4e6SMarc Zyngier static int its_vlpi_get(struct irq_data *d, struct its_cmd_info *info)
1894d011e4e6SMarc Zyngier {
1895d011e4e6SMarc Zyngier struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1896046b5054SMarc Zyngier struct its_vlpi_map *map;
1897d011e4e6SMarc Zyngier
1898046b5054SMarc Zyngier map = get_vlpi_map(d);
1899046b5054SMarc Zyngier
19005c0fb9cbSHagar Hemdan if (!its_dev->event_map.vm || !map)
19015c0fb9cbSHagar Hemdan return -EINVAL;
1902d011e4e6SMarc Zyngier
1903d011e4e6SMarc Zyngier /* Copy our mapping information to the incoming request */
1904c1d4d5cdSMarc Zyngier *info->map = *map;
1905d011e4e6SMarc Zyngier
19065c0fb9cbSHagar Hemdan return 0;
1907d011e4e6SMarc Zyngier }
1908d011e4e6SMarc Zyngier
its_vlpi_unmap(struct irq_data * d)1909d011e4e6SMarc Zyngier static int its_vlpi_unmap(struct irq_data *d)
1910d011e4e6SMarc Zyngier {
1911d011e4e6SMarc Zyngier struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1912d011e4e6SMarc Zyngier u32 event = its_get_event_id(d);
1913d011e4e6SMarc Zyngier
19145c0fb9cbSHagar Hemdan if (!its_dev->event_map.vm || !irqd_is_forwarded_to_vcpu(d))
19155c0fb9cbSHagar Hemdan return -EINVAL;
1916d011e4e6SMarc Zyngier
1917d011e4e6SMarc Zyngier /* Drop the virtual mapping */
1918d011e4e6SMarc Zyngier its_send_discard(its_dev, event);
1919d011e4e6SMarc Zyngier
1920d011e4e6SMarc Zyngier /* and restore the physical one */
1921d011e4e6SMarc Zyngier irqd_clr_forwarded_to_vcpu(d);
1922d011e4e6SMarc Zyngier its_send_mapti(its_dev, d->hwirq, event);
1923d011e4e6SMarc Zyngier lpi_update_config(d, 0xff, (LPI_PROP_DEFAULT_PRIO |
1924d011e4e6SMarc Zyngier LPI_PROP_ENABLED |
1925d011e4e6SMarc Zyngier LPI_PROP_GROUP1));
1926d011e4e6SMarc Zyngier
19272247e1bfSMarc Zyngier /* Potentially unmap the VM from this ITS */
19282247e1bfSMarc Zyngier its_unmap_vm(its_dev->its, its_dev->event_map.vm);
19292247e1bfSMarc Zyngier
1930d011e4e6SMarc Zyngier /*
1931d011e4e6SMarc Zyngier * Drop the refcount and make the device available again if
1932d011e4e6SMarc Zyngier * this was the last VLPI.
1933d011e4e6SMarc Zyngier */
1934d011e4e6SMarc Zyngier if (!--its_dev->event_map.nr_vlpis) {
1935d011e4e6SMarc Zyngier its_dev->event_map.vm = NULL;
1936d011e4e6SMarc Zyngier kfree(its_dev->event_map.vlpi_maps);
1937d011e4e6SMarc Zyngier }
1938d011e4e6SMarc Zyngier
19395c0fb9cbSHagar Hemdan return 0;
1940d011e4e6SMarc Zyngier }
1941d011e4e6SMarc Zyngier
its_vlpi_prop_update(struct irq_data * d,struct its_cmd_info * info)1942015ec038SMarc Zyngier static int its_vlpi_prop_update(struct irq_data *d, struct its_cmd_info *info)
1943015ec038SMarc Zyngier {
1944015ec038SMarc Zyngier struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1945015ec038SMarc Zyngier
1946015ec038SMarc Zyngier if (!its_dev->event_map.vm || !irqd_is_forwarded_to_vcpu(d))
1947015ec038SMarc Zyngier return -EINVAL;
1948015ec038SMarc Zyngier
1949015ec038SMarc Zyngier if (info->cmd_type == PROP_UPDATE_AND_INV_VLPI)
1950015ec038SMarc Zyngier lpi_update_config(d, 0xff, info->config);
1951015ec038SMarc Zyngier else
1952015ec038SMarc Zyngier lpi_write_config(d, 0xff, info->config);
1953015ec038SMarc Zyngier its_vlpi_set_doorbell(d, !!(info->config & LPI_PROP_ENABLED));
1954015ec038SMarc Zyngier
1955015ec038SMarc Zyngier return 0;
1956015ec038SMarc Zyngier }
1957015ec038SMarc Zyngier
its_irq_set_vcpu_affinity(struct irq_data * d,void * vcpu_info)1958c808eea8SMarc Zyngier static int its_irq_set_vcpu_affinity(struct irq_data *d, void *vcpu_info)
1959c808eea8SMarc Zyngier {
1960c808eea8SMarc Zyngier struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1961c808eea8SMarc Zyngier struct its_cmd_info *info = vcpu_info;
1962c808eea8SMarc Zyngier
1963c808eea8SMarc Zyngier /* Need a v4 ITS */
19640dd57fedSMarc Zyngier if (!is_v4(its_dev->its))
1965c808eea8SMarc Zyngier return -EINVAL;
1966c808eea8SMarc Zyngier
19675c0fb9cbSHagar Hemdan guard(raw_spinlock_irq)(&its_dev->event_map.vlpi_lock);
19685c0fb9cbSHagar Hemdan
1969d011e4e6SMarc Zyngier /* Unmap request? */
1970d011e4e6SMarc Zyngier if (!info)
1971d011e4e6SMarc Zyngier return its_vlpi_unmap(d);
1972d011e4e6SMarc Zyngier
1973c808eea8SMarc Zyngier switch (info->cmd_type) {
1974c808eea8SMarc Zyngier case MAP_VLPI:
1975d011e4e6SMarc Zyngier return its_vlpi_map(d, info);
1976c808eea8SMarc Zyngier
1977c808eea8SMarc Zyngier case GET_VLPI:
1978d011e4e6SMarc Zyngier return its_vlpi_get(d, info);
1979c808eea8SMarc Zyngier
1980c808eea8SMarc Zyngier case PROP_UPDATE_VLPI:
1981c808eea8SMarc Zyngier case PROP_UPDATE_AND_INV_VLPI:
1982015ec038SMarc Zyngier return its_vlpi_prop_update(d, info);
1983c808eea8SMarc Zyngier
1984c808eea8SMarc Zyngier default:
1985c808eea8SMarc Zyngier return -EINVAL;
1986c808eea8SMarc Zyngier }
1987c808eea8SMarc Zyngier }
1988c808eea8SMarc Zyngier
1989c48ed51cSMarc Zyngier static struct irq_chip its_irq_chip = {
1990c48ed51cSMarc Zyngier .name = "ITS",
1991c48ed51cSMarc Zyngier .irq_mask = its_mask_irq,
1992c48ed51cSMarc Zyngier .irq_unmask = its_unmask_irq,
1993004fa08dSAshok Kumar .irq_eoi = irq_chip_eoi_parent,
1994c48ed51cSMarc Zyngier .irq_set_affinity = its_set_affinity,
1995b48ac83dSMarc Zyngier .irq_compose_msi_msg = its_irq_compose_msi_msg,
19968d85dcedSMarc Zyngier .irq_set_irqchip_state = its_irq_set_irqchip_state,
19975f774f5eSMarc Zyngier .irq_retrigger = its_irq_retrigger,
1998c808eea8SMarc Zyngier .irq_set_vcpu_affinity = its_irq_set_vcpu_affinity,
1999b48ac83dSMarc Zyngier };
2000b48ac83dSMarc Zyngier
2001880cb3cdSMarc Zyngier
2002bf9529f8SMarc Zyngier /*
2003bf9529f8SMarc Zyngier * How we allocate LPIs:
2004bf9529f8SMarc Zyngier *
2005880cb3cdSMarc Zyngier * lpi_range_list contains ranges of LPIs that are to available to
2006880cb3cdSMarc Zyngier * allocate from. To allocate LPIs, just pick the first range that
2007880cb3cdSMarc Zyngier * fits the required allocation, and reduce it by the required
2008880cb3cdSMarc Zyngier * amount. Once empty, remove the range from the list.
2009bf9529f8SMarc Zyngier *
2010880cb3cdSMarc Zyngier * To free a range of LPIs, add a free range to the list, sort it and
2011880cb3cdSMarc Zyngier * merge the result if the new range happens to be adjacent to an
2012880cb3cdSMarc Zyngier * already free block.
2013880cb3cdSMarc Zyngier *
2014880cb3cdSMarc Zyngier * The consequence of the above is that allocation is cost is low, but
2015880cb3cdSMarc Zyngier * freeing is expensive. We assumes that freeing rarely occurs.
2016880cb3cdSMarc Zyngier */
20174cb205c0SJia He #define ITS_MAX_LPI_NRBITS 16 /* 64K LPIs */
2018880cb3cdSMarc Zyngier
2019880cb3cdSMarc Zyngier static DEFINE_MUTEX(lpi_range_lock);
2020880cb3cdSMarc Zyngier static LIST_HEAD(lpi_range_list);
2021bf9529f8SMarc Zyngier
2022880cb3cdSMarc Zyngier struct lpi_range {
2023880cb3cdSMarc Zyngier struct list_head entry;
2024880cb3cdSMarc Zyngier u32 base_id;
2025880cb3cdSMarc Zyngier u32 span;
2026880cb3cdSMarc Zyngier };
2027880cb3cdSMarc Zyngier
mk_lpi_range(u32 base,u32 span)2028880cb3cdSMarc Zyngier static struct lpi_range *mk_lpi_range(u32 base, u32 span)
2029bf9529f8SMarc Zyngier {
2030880cb3cdSMarc Zyngier struct lpi_range *range;
2031880cb3cdSMarc Zyngier
20321c73fac5SRasmus Villemoes range = kmalloc(sizeof(*range), GFP_KERNEL);
2033880cb3cdSMarc Zyngier if (range) {
2034880cb3cdSMarc Zyngier range->base_id = base;
2035880cb3cdSMarc Zyngier range->span = span;
2036bf9529f8SMarc Zyngier }
2037bf9529f8SMarc Zyngier
2038880cb3cdSMarc Zyngier return range;
2039880cb3cdSMarc Zyngier }
2040880cb3cdSMarc Zyngier
alloc_lpi_range(u32 nr_lpis,u32 * base)2041880cb3cdSMarc Zyngier static int alloc_lpi_range(u32 nr_lpis, u32 *base)
2042880cb3cdSMarc Zyngier {
2043880cb3cdSMarc Zyngier struct lpi_range *range, *tmp;
2044880cb3cdSMarc Zyngier int err = -ENOSPC;
2045880cb3cdSMarc Zyngier
2046880cb3cdSMarc Zyngier mutex_lock(&lpi_range_lock);
2047880cb3cdSMarc Zyngier
2048880cb3cdSMarc Zyngier list_for_each_entry_safe(range, tmp, &lpi_range_list, entry) {
2049880cb3cdSMarc Zyngier if (range->span >= nr_lpis) {
2050880cb3cdSMarc Zyngier *base = range->base_id;
2051880cb3cdSMarc Zyngier range->base_id += nr_lpis;
2052880cb3cdSMarc Zyngier range->span -= nr_lpis;
2053880cb3cdSMarc Zyngier
2054880cb3cdSMarc Zyngier if (range->span == 0) {
2055880cb3cdSMarc Zyngier list_del(&range->entry);
2056880cb3cdSMarc Zyngier kfree(range);
2057880cb3cdSMarc Zyngier }
2058880cb3cdSMarc Zyngier
2059880cb3cdSMarc Zyngier err = 0;
2060880cb3cdSMarc Zyngier break;
2061880cb3cdSMarc Zyngier }
2062880cb3cdSMarc Zyngier }
2063880cb3cdSMarc Zyngier
2064880cb3cdSMarc Zyngier mutex_unlock(&lpi_range_lock);
2065880cb3cdSMarc Zyngier
2066880cb3cdSMarc Zyngier pr_debug("ITS: alloc %u:%u\n", *base, nr_lpis);
2067880cb3cdSMarc Zyngier return err;
2068880cb3cdSMarc Zyngier }
2069880cb3cdSMarc Zyngier
merge_lpi_ranges(struct lpi_range * a,struct lpi_range * b)207012eade12SRasmus Villemoes static void merge_lpi_ranges(struct lpi_range *a, struct lpi_range *b)
207112eade12SRasmus Villemoes {
207212eade12SRasmus Villemoes if (&a->entry == &lpi_range_list || &b->entry == &lpi_range_list)
207312eade12SRasmus Villemoes return;
207412eade12SRasmus Villemoes if (a->base_id + a->span != b->base_id)
207512eade12SRasmus Villemoes return;
207612eade12SRasmus Villemoes b->base_id = a->base_id;
207712eade12SRasmus Villemoes b->span += a->span;
207812eade12SRasmus Villemoes list_del(&a->entry);
207912eade12SRasmus Villemoes kfree(a);
208012eade12SRasmus Villemoes }
208112eade12SRasmus Villemoes
free_lpi_range(u32 base,u32 nr_lpis)2082880cb3cdSMarc Zyngier static int free_lpi_range(u32 base, u32 nr_lpis)
2083880cb3cdSMarc Zyngier {
208412eade12SRasmus Villemoes struct lpi_range *new, *old;
2085880cb3cdSMarc Zyngier
2086880cb3cdSMarc Zyngier new = mk_lpi_range(base, nr_lpis);
2087b31a3838SRasmus Villemoes if (!new)
2088b31a3838SRasmus Villemoes return -ENOMEM;
2089880cb3cdSMarc Zyngier
2090880cb3cdSMarc Zyngier mutex_lock(&lpi_range_lock);
2091880cb3cdSMarc Zyngier
209212eade12SRasmus Villemoes list_for_each_entry_reverse(old, &lpi_range_list, entry) {
209312eade12SRasmus Villemoes if (old->base_id < base)
209412eade12SRasmus Villemoes break;
2095880cb3cdSMarc Zyngier }
209612eade12SRasmus Villemoes /*
209712eade12SRasmus Villemoes * old is the last element with ->base_id smaller than base,
209812eade12SRasmus Villemoes * so new goes right after it. If there are no elements with
209912eade12SRasmus Villemoes * ->base_id smaller than base, &old->entry ends up pointing
210012eade12SRasmus Villemoes * at the head of the list, and inserting new it the start of
210112eade12SRasmus Villemoes * the list is the right thing to do in that case as well.
210212eade12SRasmus Villemoes */
210312eade12SRasmus Villemoes list_add(&new->entry, &old->entry);
210412eade12SRasmus Villemoes /*
210512eade12SRasmus Villemoes * Now check if we can merge with the preceding and/or
210612eade12SRasmus Villemoes * following ranges.
210712eade12SRasmus Villemoes */
210812eade12SRasmus Villemoes merge_lpi_ranges(old, new);
210912eade12SRasmus Villemoes merge_lpi_ranges(new, list_next_entry(new, entry));
2110880cb3cdSMarc Zyngier
2111880cb3cdSMarc Zyngier mutex_unlock(&lpi_range_lock);
2112b31a3838SRasmus Villemoes return 0;
2113bf9529f8SMarc Zyngier }
2114bf9529f8SMarc Zyngier
its_lpi_init(u32 id_bits)211504a0e4deSTomasz Nowicki static int __init its_lpi_init(u32 id_bits)
2116bf9529f8SMarc Zyngier {
2117880cb3cdSMarc Zyngier u32 lpis = (1UL << id_bits) - 8192;
211812b2905aSMarc Zyngier u32 numlpis;
2119880cb3cdSMarc Zyngier int err;
2120bf9529f8SMarc Zyngier
212112b2905aSMarc Zyngier numlpis = 1UL << GICD_TYPER_NUM_LPIS(gic_rdists->gicd_typer);
212212b2905aSMarc Zyngier
212312b2905aSMarc Zyngier if (numlpis > 2 && !WARN_ON(numlpis > lpis)) {
212412b2905aSMarc Zyngier lpis = numlpis;
212512b2905aSMarc Zyngier pr_info("ITS: Using hypervisor restricted LPI range [%u]\n",
212612b2905aSMarc Zyngier lpis);
212712b2905aSMarc Zyngier }
212812b2905aSMarc Zyngier
2129880cb3cdSMarc Zyngier /*
2130880cb3cdSMarc Zyngier * Initializing the allocator is just the same as freeing the
2131880cb3cdSMarc Zyngier * full range of LPIs.
2132880cb3cdSMarc Zyngier */
2133880cb3cdSMarc Zyngier err = free_lpi_range(8192, lpis);
2134880cb3cdSMarc Zyngier pr_debug("ITS: Allocator initialized for %u LPIs\n", lpis);
2135880cb3cdSMarc Zyngier return err;
2136bf9529f8SMarc Zyngier }
2137bf9529f8SMarc Zyngier
its_lpi_alloc(int nr_irqs,u32 * base,int * nr_ids)213838dd7c49SMarc Zyngier static unsigned long *its_lpi_alloc(int nr_irqs, u32 *base, int *nr_ids)
2139bf9529f8SMarc Zyngier {
2140bf9529f8SMarc Zyngier unsigned long *bitmap = NULL;
2141880cb3cdSMarc Zyngier int err = 0;
2142bf9529f8SMarc Zyngier
2143bf9529f8SMarc Zyngier do {
214438dd7c49SMarc Zyngier err = alloc_lpi_range(nr_irqs, base);
2145880cb3cdSMarc Zyngier if (!err)
2146bf9529f8SMarc Zyngier break;
2147bf9529f8SMarc Zyngier
214838dd7c49SMarc Zyngier nr_irqs /= 2;
214938dd7c49SMarc Zyngier } while (nr_irqs > 0);
2150bf9529f8SMarc Zyngier
215145725e0fSMarc Zyngier if (!nr_irqs)
215245725e0fSMarc Zyngier err = -ENOSPC;
215345725e0fSMarc Zyngier
2154880cb3cdSMarc Zyngier if (err)
2155bf9529f8SMarc Zyngier goto out;
2156bf9529f8SMarc Zyngier
2157ff5fe886SAndy Shevchenko bitmap = bitmap_zalloc(nr_irqs, GFP_ATOMIC);
2158bf9529f8SMarc Zyngier if (!bitmap)
2159bf9529f8SMarc Zyngier goto out;
2160bf9529f8SMarc Zyngier
216138dd7c49SMarc Zyngier *nr_ids = nr_irqs;
2162bf9529f8SMarc Zyngier
2163bf9529f8SMarc Zyngier out:
2164c8415b94SMarc Zyngier if (!bitmap)
2165c8415b94SMarc Zyngier *base = *nr_ids = 0;
2166c8415b94SMarc Zyngier
2167bf9529f8SMarc Zyngier return bitmap;
2168bf9529f8SMarc Zyngier }
2169bf9529f8SMarc Zyngier
its_lpi_free(unsigned long * bitmap,u32 base,u32 nr_ids)217038dd7c49SMarc Zyngier static void its_lpi_free(unsigned long *bitmap, u32 base, u32 nr_ids)
2171bf9529f8SMarc Zyngier {
2172880cb3cdSMarc Zyngier WARN_ON(free_lpi_range(base, nr_ids));
2173ff5fe886SAndy Shevchenko bitmap_free(bitmap);
2174bf9529f8SMarc Zyngier }
21751ac19ca6SMarc Zyngier
gic_reset_prop_table(void * va)2176053be485SMarc Zyngier static void gic_reset_prop_table(void *va)
2177053be485SMarc Zyngier {
2178053be485SMarc Zyngier /* Priority 0xa0, Group-1, disabled */
2179053be485SMarc Zyngier memset(va, LPI_PROP_DEFAULT_PRIO | LPI_PROP_GROUP1, LPI_PROPBASE_SZ);
2180053be485SMarc Zyngier
2181053be485SMarc Zyngier /* Make sure the GIC will observe the written configuration */
2182053be485SMarc Zyngier gic_flush_dcache_to_poc(va, LPI_PROPBASE_SZ);
2183053be485SMarc Zyngier }
2184053be485SMarc Zyngier
its_allocate_prop_table(gfp_t gfp_flags)21850e5ccf91SMarc Zyngier static struct page *its_allocate_prop_table(gfp_t gfp_flags)
21860e5ccf91SMarc Zyngier {
21870e5ccf91SMarc Zyngier struct page *prop_page;
21881ac19ca6SMarc Zyngier
21890e5ccf91SMarc Zyngier prop_page = alloc_pages(gfp_flags, get_order(LPI_PROPBASE_SZ));
21900e5ccf91SMarc Zyngier if (!prop_page)
21910e5ccf91SMarc Zyngier return NULL;
21920e5ccf91SMarc Zyngier
2193053be485SMarc Zyngier gic_reset_prop_table(page_address(prop_page));
21940e5ccf91SMarc Zyngier
21950e5ccf91SMarc Zyngier return prop_page;
21960e5ccf91SMarc Zyngier }
21970e5ccf91SMarc Zyngier
its_free_prop_table(struct page * prop_page)21987d75bbb4SMarc Zyngier static void its_free_prop_table(struct page *prop_page)
21997d75bbb4SMarc Zyngier {
22007d75bbb4SMarc Zyngier free_pages((unsigned long)page_address(prop_page),
22017d75bbb4SMarc Zyngier get_order(LPI_PROPBASE_SZ));
22027d75bbb4SMarc Zyngier }
22031ac19ca6SMarc Zyngier
gic_check_reserved_range(phys_addr_t addr,unsigned long size)22045e2c9f9aSMarc Zyngier static bool gic_check_reserved_range(phys_addr_t addr, unsigned long size)
22055e2c9f9aSMarc Zyngier {
22065e2c9f9aSMarc Zyngier phys_addr_t start, end, addr_end;
22075e2c9f9aSMarc Zyngier u64 i;
22085e2c9f9aSMarc Zyngier
22095e2c9f9aSMarc Zyngier /*
22105e2c9f9aSMarc Zyngier * We don't bother checking for a kdump kernel as by
22115e2c9f9aSMarc Zyngier * construction, the LPI tables are out of this kernel's
22125e2c9f9aSMarc Zyngier * memory map.
22135e2c9f9aSMarc Zyngier */
22145e2c9f9aSMarc Zyngier if (is_kdump_kernel())
22155e2c9f9aSMarc Zyngier return true;
22165e2c9f9aSMarc Zyngier
22175e2c9f9aSMarc Zyngier addr_end = addr + size - 1;
22185e2c9f9aSMarc Zyngier
22199f3d5eaaSMike Rapoport for_each_reserved_mem_range(i, &start, &end) {
22205e2c9f9aSMarc Zyngier if (addr >= start && addr_end <= end)
22215e2c9f9aSMarc Zyngier return true;
22225e2c9f9aSMarc Zyngier }
22235e2c9f9aSMarc Zyngier
22245e2c9f9aSMarc Zyngier /* Not found, not a good sign... */
22255e2c9f9aSMarc Zyngier pr_warn("GICv3: Expected reserved range [%pa:%pa], not found\n",
22265e2c9f9aSMarc Zyngier &addr, &addr_end);
22275e2c9f9aSMarc Zyngier add_taint(TAINT_CRAP, LOCKDEP_STILL_OK);
22285e2c9f9aSMarc Zyngier return false;
22295e2c9f9aSMarc Zyngier }
22305e2c9f9aSMarc Zyngier
gic_reserve_range(phys_addr_t addr,unsigned long size)22313fb68faeSMarc Zyngier static int gic_reserve_range(phys_addr_t addr, unsigned long size)
22323fb68faeSMarc Zyngier {
22333fb68faeSMarc Zyngier if (efi_enabled(EFI_CONFIG_TABLES))
22343fb68faeSMarc Zyngier return efi_mem_reserve_persistent(addr, size);
22353fb68faeSMarc Zyngier
22363fb68faeSMarc Zyngier return 0;
22373fb68faeSMarc Zyngier }
22383fb68faeSMarc Zyngier
its_setup_lpi_prop_table(void)223911e37d35SMarc Zyngier static int __init its_setup_lpi_prop_table(void)
22401ac19ca6SMarc Zyngier {
2241c440a9d9SMarc Zyngier if (gic_rdists->flags & RDIST_FLAGS_RD_TABLES_PREALLOCATED) {
2242c440a9d9SMarc Zyngier u64 val;
2243c440a9d9SMarc Zyngier
2244c440a9d9SMarc Zyngier val = gicr_read_propbaser(gic_data_rdist_rd_base() + GICR_PROPBASER);
2245c440a9d9SMarc Zyngier lpi_id_bits = (val & GICR_PROPBASER_IDBITS_MASK) + 1;
2246c440a9d9SMarc Zyngier
2247c440a9d9SMarc Zyngier gic_rdists->prop_table_pa = val & GENMASK_ULL(51, 12);
2248c440a9d9SMarc Zyngier gic_rdists->prop_table_va = memremap(gic_rdists->prop_table_pa,
2249c440a9d9SMarc Zyngier LPI_PROPBASE_SZ,
2250c440a9d9SMarc Zyngier MEMREMAP_WB);
2251c440a9d9SMarc Zyngier gic_reset_prop_table(gic_rdists->prop_table_va);
2252c440a9d9SMarc Zyngier } else {
2253e1a2e201SMarc Zyngier struct page *page;
22541ac19ca6SMarc Zyngier
2255c440a9d9SMarc Zyngier lpi_id_bits = min_t(u32,
2256c440a9d9SMarc Zyngier GICD_TYPER_ID_BITS(gic_rdists->gicd_typer),
22574cb205c0SJia He ITS_MAX_LPI_NRBITS);
2258e1a2e201SMarc Zyngier page = its_allocate_prop_table(GFP_NOWAIT);
2259e1a2e201SMarc Zyngier if (!page) {
22601ac19ca6SMarc Zyngier pr_err("Failed to allocate PROPBASE\n");
22611ac19ca6SMarc Zyngier return -ENOMEM;
22621ac19ca6SMarc Zyngier }
22631ac19ca6SMarc Zyngier
2264e1a2e201SMarc Zyngier gic_rdists->prop_table_pa = page_to_phys(page);
2265e1a2e201SMarc Zyngier gic_rdists->prop_table_va = page_address(page);
22663fb68faeSMarc Zyngier WARN_ON(gic_reserve_range(gic_rdists->prop_table_pa,
22673fb68faeSMarc Zyngier LPI_PROPBASE_SZ));
2268c440a9d9SMarc Zyngier }
2269e1a2e201SMarc Zyngier
2270e1a2e201SMarc Zyngier pr_info("GICv3: using LPI property table @%pa\n",
2271e1a2e201SMarc Zyngier &gic_rdists->prop_table_pa);
22721ac19ca6SMarc Zyngier
22736c31e123SShanker Donthineni return its_lpi_init(lpi_id_bits);
22741ac19ca6SMarc Zyngier }
22751ac19ca6SMarc Zyngier
22761ac19ca6SMarc Zyngier static const char *its_base_type_string[] = {
22771ac19ca6SMarc Zyngier [GITS_BASER_TYPE_DEVICE] = "Devices",
22781ac19ca6SMarc Zyngier [GITS_BASER_TYPE_VCPU] = "Virtual CPUs",
22794f46de9dSMarc Zyngier [GITS_BASER_TYPE_RESERVED3] = "Reserved (3)",
22801ac19ca6SMarc Zyngier [GITS_BASER_TYPE_COLLECTION] = "Interrupt Collections",
22811ac19ca6SMarc Zyngier [GITS_BASER_TYPE_RESERVED5] = "Reserved (5)",
22821ac19ca6SMarc Zyngier [GITS_BASER_TYPE_RESERVED6] = "Reserved (6)",
22831ac19ca6SMarc Zyngier [GITS_BASER_TYPE_RESERVED7] = "Reserved (7)",
22841ac19ca6SMarc Zyngier };
22851ac19ca6SMarc Zyngier
its_read_baser(struct its_node * its,struct its_baser * baser)22862d81d425SShanker Donthineni static u64 its_read_baser(struct its_node *its, struct its_baser *baser)
22872d81d425SShanker Donthineni {
22882d81d425SShanker Donthineni u32 idx = baser - its->tables;
22892d81d425SShanker Donthineni
22900968a619SVladimir Murzin return gits_read_baser(its->base + GITS_BASER + (idx << 3));
22912d81d425SShanker Donthineni }
22922d81d425SShanker Donthineni
its_write_baser(struct its_node * its,struct its_baser * baser,u64 val)22932d81d425SShanker Donthineni static void its_write_baser(struct its_node *its, struct its_baser *baser,
22942d81d425SShanker Donthineni u64 val)
22952d81d425SShanker Donthineni {
22962d81d425SShanker Donthineni u32 idx = baser - its->tables;
22972d81d425SShanker Donthineni
22980968a619SVladimir Murzin gits_write_baser(val, its->base + GITS_BASER + (idx << 3));
22992d81d425SShanker Donthineni baser->val = its_read_baser(its, baser);
23002d81d425SShanker Donthineni }
23012d81d425SShanker Donthineni
its_setup_baser(struct its_node * its,struct its_baser * baser,u64 cache,u64 shr,u32 order,bool indirect)23029347359aSShanker Donthineni static int its_setup_baser(struct its_node *its, struct its_baser *baser,
2303d5df9dc9SMarc Zyngier u64 cache, u64 shr, u32 order, bool indirect)
23049347359aSShanker Donthineni {
23059347359aSShanker Donthineni u64 val = its_read_baser(its, baser);
23069347359aSShanker Donthineni u64 esz = GITS_BASER_ENTRY_SIZE(val);
23079347359aSShanker Donthineni u64 type = GITS_BASER_TYPE(val);
230830ae9610SShanker Donthineni u64 baser_phys, tmp;
2309d5df9dc9SMarc Zyngier u32 alloc_pages, psz;
2310539d3782SShanker Donthineni struct page *page;
23119347359aSShanker Donthineni void *base;
23129347359aSShanker Donthineni
2313d5df9dc9SMarc Zyngier psz = baser->psz;
23149347359aSShanker Donthineni alloc_pages = (PAGE_ORDER_TO_SIZE(order) / psz);
23159347359aSShanker Donthineni if (alloc_pages > GITS_BASER_PAGES_MAX) {
23169347359aSShanker Donthineni pr_warn("ITS@%pa: %s too large, reduce ITS pages %u->%u\n",
23179347359aSShanker Donthineni &its->phys_base, its_base_type_string[type],
23189347359aSShanker Donthineni alloc_pages, GITS_BASER_PAGES_MAX);
23199347359aSShanker Donthineni alloc_pages = GITS_BASER_PAGES_MAX;
23209347359aSShanker Donthineni order = get_order(GITS_BASER_PAGES_MAX * psz);
23219347359aSShanker Donthineni }
23229347359aSShanker Donthineni
2323539d3782SShanker Donthineni page = alloc_pages_node(its->numa_node, GFP_KERNEL | __GFP_ZERO, order);
2324539d3782SShanker Donthineni if (!page)
23259347359aSShanker Donthineni return -ENOMEM;
23269347359aSShanker Donthineni
2327539d3782SShanker Donthineni base = (void *)page_address(page);
232830ae9610SShanker Donthineni baser_phys = virt_to_phys(base);
232930ae9610SShanker Donthineni
233030ae9610SShanker Donthineni /* Check if the physical address of the memory is above 48bits */
233130ae9610SShanker Donthineni if (IS_ENABLED(CONFIG_ARM64_64K_PAGES) && (baser_phys >> 48)) {
233230ae9610SShanker Donthineni
233330ae9610SShanker Donthineni /* 52bit PA is supported only when PageSize=64K */
233430ae9610SShanker Donthineni if (psz != SZ_64K) {
233530ae9610SShanker Donthineni pr_err("ITS: no 52bit PA support when psz=%d\n", psz);
233630ae9610SShanker Donthineni free_pages((unsigned long)base, order);
233730ae9610SShanker Donthineni return -ENXIO;
233830ae9610SShanker Donthineni }
233930ae9610SShanker Donthineni
234030ae9610SShanker Donthineni /* Convert 52bit PA to 48bit field */
234130ae9610SShanker Donthineni baser_phys = GITS_BASER_PHYS_52_to_48(baser_phys);
234230ae9610SShanker Donthineni }
234330ae9610SShanker Donthineni
23449347359aSShanker Donthineni retry_baser:
234530ae9610SShanker Donthineni val = (baser_phys |
23469347359aSShanker Donthineni (type << GITS_BASER_TYPE_SHIFT) |
23479347359aSShanker Donthineni ((esz - 1) << GITS_BASER_ENTRY_SIZE_SHIFT) |
23489347359aSShanker Donthineni ((alloc_pages - 1) << GITS_BASER_PAGES_SHIFT) |
23499347359aSShanker Donthineni cache |
23509347359aSShanker Donthineni shr |
23519347359aSShanker Donthineni GITS_BASER_VALID);
23529347359aSShanker Donthineni
23533faf24eaSShanker Donthineni val |= indirect ? GITS_BASER_INDIRECT : 0x0;
23543faf24eaSShanker Donthineni
23559347359aSShanker Donthineni switch (psz) {
23569347359aSShanker Donthineni case SZ_4K:
23579347359aSShanker Donthineni val |= GITS_BASER_PAGE_SIZE_4K;
23589347359aSShanker Donthineni break;
23599347359aSShanker Donthineni case SZ_16K:
23609347359aSShanker Donthineni val |= GITS_BASER_PAGE_SIZE_16K;
23619347359aSShanker Donthineni break;
23629347359aSShanker Donthineni case SZ_64K:
23639347359aSShanker Donthineni val |= GITS_BASER_PAGE_SIZE_64K;
23649347359aSShanker Donthineni break;
23659347359aSShanker Donthineni }
23669347359aSShanker Donthineni
23673fca09b6SFang Xiang if (!shr)
23683fca09b6SFang Xiang gic_flush_dcache_to_poc(base, PAGE_ORDER_TO_SIZE(order));
23693fca09b6SFang Xiang
23709347359aSShanker Donthineni its_write_baser(its, baser, val);
23719347359aSShanker Donthineni tmp = baser->val;
23729347359aSShanker Donthineni
23739347359aSShanker Donthineni if ((val ^ tmp) & GITS_BASER_SHAREABILITY_MASK) {
23749347359aSShanker Donthineni /*
23759347359aSShanker Donthineni * Shareability didn't stick. Just use
23769347359aSShanker Donthineni * whatever the read reported, which is likely
23779347359aSShanker Donthineni * to be the only thing this redistributor
23789347359aSShanker Donthineni * supports. If that's zero, make it
23799347359aSShanker Donthineni * non-cacheable as well.
23809347359aSShanker Donthineni */
23819347359aSShanker Donthineni shr = tmp & GITS_BASER_SHAREABILITY_MASK;
23823fca09b6SFang Xiang if (!shr)
23839347359aSShanker Donthineni cache = GITS_BASER_nC;
23843fca09b6SFang Xiang
23859347359aSShanker Donthineni goto retry_baser;
23869347359aSShanker Donthineni }
23879347359aSShanker Donthineni
23889347359aSShanker Donthineni if (val != tmp) {
2389b11283ebSVladimir Murzin pr_err("ITS@%pa: %s doesn't stick: %llx %llx\n",
23909347359aSShanker Donthineni &its->phys_base, its_base_type_string[type],
2391b11283ebSVladimir Murzin val, tmp);
23929347359aSShanker Donthineni free_pages((unsigned long)base, order);
23939347359aSShanker Donthineni return -ENXIO;
23949347359aSShanker Donthineni }
23959347359aSShanker Donthineni
23969347359aSShanker Donthineni baser->order = order;
23979347359aSShanker Donthineni baser->base = base;
23989347359aSShanker Donthineni baser->psz = psz;
23993faf24eaSShanker Donthineni tmp = indirect ? GITS_LVL1_ENTRY_SIZE : esz;
24009347359aSShanker Donthineni
24013faf24eaSShanker Donthineni pr_info("ITS@%pa: allocated %d %s @%lx (%s, esz %d, psz %dK, shr %d)\n",
2402d524eaa2SVladimir Murzin &its->phys_base, (int)(PAGE_ORDER_TO_SIZE(order) / (int)tmp),
24039347359aSShanker Donthineni its_base_type_string[type],
24049347359aSShanker Donthineni (unsigned long)virt_to_phys(base),
24053faf24eaSShanker Donthineni indirect ? "indirect" : "flat", (int)esz,
24069347359aSShanker Donthineni psz / SZ_1K, (int)shr >> GITS_BASER_SHAREABILITY_SHIFT);
24079347359aSShanker Donthineni
24089347359aSShanker Donthineni return 0;
24099347359aSShanker Donthineni }
24109347359aSShanker Donthineni
its_parse_indirect_baser(struct its_node * its,struct its_baser * baser,u32 * order,u32 ids)24114cacac57SMarc Zyngier static bool its_parse_indirect_baser(struct its_node *its,
24124cacac57SMarc Zyngier struct its_baser *baser,
2413d5df9dc9SMarc Zyngier u32 *order, u32 ids)
24144b75c459SShanker Donthineni {
24154cacac57SMarc Zyngier u64 tmp = its_read_baser(its, baser);
24164cacac57SMarc Zyngier u64 type = GITS_BASER_TYPE(tmp);
24174cacac57SMarc Zyngier u64 esz = GITS_BASER_ENTRY_SIZE(tmp);
24182fd632a0SShanker Donthineni u64 val = GITS_BASER_InnerShareable | GITS_BASER_RaWaWb;
24194b75c459SShanker Donthineni u32 new_order = *order;
2420d5df9dc9SMarc Zyngier u32 psz = baser->psz;
24213faf24eaSShanker Donthineni bool indirect = false;
24223faf24eaSShanker Donthineni
24233faf24eaSShanker Donthineni /* No need to enable Indirection if memory requirement < (psz*2)bytes */
24243faf24eaSShanker Donthineni if ((esz << ids) > (psz * 2)) {
24253faf24eaSShanker Donthineni /*
24263faf24eaSShanker Donthineni * Find out whether hw supports a single or two-level table by
24273faf24eaSShanker Donthineni * table by reading bit at offset '62' after writing '1' to it.
24283faf24eaSShanker Donthineni */
24293faf24eaSShanker Donthineni its_write_baser(its, baser, val | GITS_BASER_INDIRECT);
24303faf24eaSShanker Donthineni indirect = !!(baser->val & GITS_BASER_INDIRECT);
24313faf24eaSShanker Donthineni
24323faf24eaSShanker Donthineni if (indirect) {
24333faf24eaSShanker Donthineni /*
24343faf24eaSShanker Donthineni * The size of the lvl2 table is equal to ITS page size
24353faf24eaSShanker Donthineni * which is 'psz'. For computing lvl1 table size,
24363faf24eaSShanker Donthineni * subtract ID bits that sparse lvl2 table from 'ids'
24373faf24eaSShanker Donthineni * which is reported by ITS hardware times lvl1 table
24383faf24eaSShanker Donthineni * entry size.
24393faf24eaSShanker Donthineni */
2440d524eaa2SVladimir Murzin ids -= ilog2(psz / (int)esz);
24413faf24eaSShanker Donthineni esz = GITS_LVL1_ENTRY_SIZE;
24423faf24eaSShanker Donthineni }
24433faf24eaSShanker Donthineni }
24444b75c459SShanker Donthineni
24454b75c459SShanker Donthineni /*
24464b75c459SShanker Donthineni * Allocate as many entries as required to fit the
24474b75c459SShanker Donthineni * range of device IDs that the ITS can grok... The ID
24484b75c459SShanker Donthineni * space being incredibly sparse, this results in a
24493faf24eaSShanker Donthineni * massive waste of memory if two-level device table
24503faf24eaSShanker Donthineni * feature is not supported by hardware.
24514b75c459SShanker Donthineni */
24524b75c459SShanker Donthineni new_order = max_t(u32, get_order(esz << ids), new_order);
245323baf831SKirill A. Shutemov if (new_order > MAX_ORDER) {
245423baf831SKirill A. Shutemov new_order = MAX_ORDER;
2455d524eaa2SVladimir Murzin ids = ilog2(PAGE_ORDER_TO_SIZE(new_order) / (int)esz);
2456576a8342SMarc Zyngier pr_warn("ITS@%pa: %s Table too large, reduce ids %llu->%u\n",
24574cacac57SMarc Zyngier &its->phys_base, its_base_type_string[type],
2458576a8342SMarc Zyngier device_ids(its), ids);
24594b75c459SShanker Donthineni }
24604b75c459SShanker Donthineni
24614b75c459SShanker Donthineni *order = new_order;
24623faf24eaSShanker Donthineni
24633faf24eaSShanker Donthineni return indirect;
24644b75c459SShanker Donthineni }
24654b75c459SShanker Donthineni
compute_common_aff(u64 val)24665e516846SMarc Zyngier static u32 compute_common_aff(u64 val)
24675e516846SMarc Zyngier {
24685e516846SMarc Zyngier u32 aff, clpiaff;
24695e516846SMarc Zyngier
24705e516846SMarc Zyngier aff = FIELD_GET(GICR_TYPER_AFFINITY, val);
24715e516846SMarc Zyngier clpiaff = FIELD_GET(GICR_TYPER_COMMON_LPI_AFF, val);
24725e516846SMarc Zyngier
24735e516846SMarc Zyngier return aff & ~(GENMASK(31, 0) >> (clpiaff * 8));
24745e516846SMarc Zyngier }
24755e516846SMarc Zyngier
compute_its_aff(struct its_node * its)24765e516846SMarc Zyngier static u32 compute_its_aff(struct its_node *its)
24775e516846SMarc Zyngier {
24785e516846SMarc Zyngier u64 val;
24795e516846SMarc Zyngier u32 svpet;
24805e516846SMarc Zyngier
24815e516846SMarc Zyngier /*
24825e516846SMarc Zyngier * Reencode the ITS SVPET and MPIDR as a GICR_TYPER, and compute
24835e516846SMarc Zyngier * the resulting affinity. We then use that to see if this match
24845e516846SMarc Zyngier * our own affinity.
24855e516846SMarc Zyngier */
24865e516846SMarc Zyngier svpet = FIELD_GET(GITS_TYPER_SVPET, its->typer);
24875e516846SMarc Zyngier val = FIELD_PREP(GICR_TYPER_COMMON_LPI_AFF, svpet);
24885e516846SMarc Zyngier val |= FIELD_PREP(GICR_TYPER_AFFINITY, its->mpidr);
24895e516846SMarc Zyngier return compute_common_aff(val);
24905e516846SMarc Zyngier }
24915e516846SMarc Zyngier
find_sibling_its(struct its_node * cur_its)24925e516846SMarc Zyngier static struct its_node *find_sibling_its(struct its_node *cur_its)
24935e516846SMarc Zyngier {
24945e516846SMarc Zyngier struct its_node *its;
24955e516846SMarc Zyngier u32 aff;
24965e516846SMarc Zyngier
24975e516846SMarc Zyngier if (!FIELD_GET(GITS_TYPER_SVPET, cur_its->typer))
24985e516846SMarc Zyngier return NULL;
24995e516846SMarc Zyngier
25005e516846SMarc Zyngier aff = compute_its_aff(cur_its);
25015e516846SMarc Zyngier
25025e516846SMarc Zyngier list_for_each_entry(its, &its_nodes, entry) {
25035e516846SMarc Zyngier u64 baser;
25045e516846SMarc Zyngier
25055e516846SMarc Zyngier if (!is_v4_1(its) || its == cur_its)
25065e516846SMarc Zyngier continue;
25075e516846SMarc Zyngier
25085e516846SMarc Zyngier if (!FIELD_GET(GITS_TYPER_SVPET, its->typer))
25095e516846SMarc Zyngier continue;
25105e516846SMarc Zyngier
25115e516846SMarc Zyngier if (aff != compute_its_aff(its))
25125e516846SMarc Zyngier continue;
25135e516846SMarc Zyngier
25145e516846SMarc Zyngier /* GICv4.1 guarantees that the vPE table is GITS_BASER2 */
25155e516846SMarc Zyngier baser = its->tables[2].val;
25165e516846SMarc Zyngier if (!(baser & GITS_BASER_VALID))
25175e516846SMarc Zyngier continue;
25185e516846SMarc Zyngier
25195e516846SMarc Zyngier return its;
25205e516846SMarc Zyngier }
25215e516846SMarc Zyngier
25225e516846SMarc Zyngier return NULL;
25235e516846SMarc Zyngier }
25245e516846SMarc Zyngier
its_free_tables(struct its_node * its)25251ac19ca6SMarc Zyngier static void its_free_tables(struct its_node *its)
25261ac19ca6SMarc Zyngier {
25271ac19ca6SMarc Zyngier int i;
25281ac19ca6SMarc Zyngier
25291ac19ca6SMarc Zyngier for (i = 0; i < GITS_BASER_NR_REGS; i++) {
25301a485f4dSShanker Donthineni if (its->tables[i].base) {
25311a485f4dSShanker Donthineni free_pages((unsigned long)its->tables[i].base,
25321a485f4dSShanker Donthineni its->tables[i].order);
25331a485f4dSShanker Donthineni its->tables[i].base = NULL;
25341ac19ca6SMarc Zyngier }
25351ac19ca6SMarc Zyngier }
25361ac19ca6SMarc Zyngier }
25371ac19ca6SMarc Zyngier
its_probe_baser_psz(struct its_node * its,struct its_baser * baser)2538d5df9dc9SMarc Zyngier static int its_probe_baser_psz(struct its_node *its, struct its_baser *baser)
2539d5df9dc9SMarc Zyngier {
2540d5df9dc9SMarc Zyngier u64 psz = SZ_64K;
2541d5df9dc9SMarc Zyngier
2542d5df9dc9SMarc Zyngier while (psz) {
2543d5df9dc9SMarc Zyngier u64 val, gpsz;
2544d5df9dc9SMarc Zyngier
2545d5df9dc9SMarc Zyngier val = its_read_baser(its, baser);
2546d5df9dc9SMarc Zyngier val &= ~GITS_BASER_PAGE_SIZE_MASK;
2547d5df9dc9SMarc Zyngier
2548d5df9dc9SMarc Zyngier switch (psz) {
2549d5df9dc9SMarc Zyngier case SZ_64K:
2550d5df9dc9SMarc Zyngier gpsz = GITS_BASER_PAGE_SIZE_64K;
2551d5df9dc9SMarc Zyngier break;
2552d5df9dc9SMarc Zyngier case SZ_16K:
2553d5df9dc9SMarc Zyngier gpsz = GITS_BASER_PAGE_SIZE_16K;
2554d5df9dc9SMarc Zyngier break;
2555d5df9dc9SMarc Zyngier case SZ_4K:
2556d5df9dc9SMarc Zyngier default:
2557d5df9dc9SMarc Zyngier gpsz = GITS_BASER_PAGE_SIZE_4K;
2558d5df9dc9SMarc Zyngier break;
2559d5df9dc9SMarc Zyngier }
2560d5df9dc9SMarc Zyngier
2561d5df9dc9SMarc Zyngier gpsz >>= GITS_BASER_PAGE_SIZE_SHIFT;
2562d5df9dc9SMarc Zyngier
2563d5df9dc9SMarc Zyngier val |= FIELD_PREP(GITS_BASER_PAGE_SIZE_MASK, gpsz);
2564d5df9dc9SMarc Zyngier its_write_baser(its, baser, val);
2565d5df9dc9SMarc Zyngier
2566d5df9dc9SMarc Zyngier if (FIELD_GET(GITS_BASER_PAGE_SIZE_MASK, baser->val) == gpsz)
2567d5df9dc9SMarc Zyngier break;
2568d5df9dc9SMarc Zyngier
2569d5df9dc9SMarc Zyngier switch (psz) {
2570d5df9dc9SMarc Zyngier case SZ_64K:
2571d5df9dc9SMarc Zyngier psz = SZ_16K;
2572d5df9dc9SMarc Zyngier break;
2573d5df9dc9SMarc Zyngier case SZ_16K:
2574d5df9dc9SMarc Zyngier psz = SZ_4K;
2575d5df9dc9SMarc Zyngier break;
2576d5df9dc9SMarc Zyngier case SZ_4K:
2577d5df9dc9SMarc Zyngier default:
2578d5df9dc9SMarc Zyngier return -1;
2579d5df9dc9SMarc Zyngier }
2580d5df9dc9SMarc Zyngier }
2581d5df9dc9SMarc Zyngier
2582d5df9dc9SMarc Zyngier baser->psz = psz;
2583d5df9dc9SMarc Zyngier return 0;
2584d5df9dc9SMarc Zyngier }
2585d5df9dc9SMarc Zyngier
its_alloc_tables(struct its_node * its)25860e0b0f69SShanker Donthineni static int its_alloc_tables(struct its_node *its)
25871ac19ca6SMarc Zyngier {
25881ac19ca6SMarc Zyngier u64 shr = GITS_BASER_InnerShareable;
25892fd632a0SShanker Donthineni u64 cache = GITS_BASER_RaWaWb;
25909347359aSShanker Donthineni int err, i;
259194100970SRobert Richter
2592fa150019SArd Biesheuvel if (its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_22375)
2593fa150019SArd Biesheuvel /* erratum 24313: ignore memory access type */
25949347359aSShanker Donthineni cache = GITS_BASER_nCnB;
2595466b7d16SShanker Donthineni
25963fca09b6SFang Xiang if (its->flags & ITS_FLAGS_FORCE_NON_SHAREABLE) {
25973fca09b6SFang Xiang cache = GITS_BASER_nC;
25983fca09b6SFang Xiang shr = 0;
25993fca09b6SFang Xiang }
26003fca09b6SFang Xiang
26011ac19ca6SMarc Zyngier for (i = 0; i < GITS_BASER_NR_REGS; i++) {
26022d81d425SShanker Donthineni struct its_baser *baser = its->tables + i;
26032d81d425SShanker Donthineni u64 val = its_read_baser(its, baser);
26041ac19ca6SMarc Zyngier u64 type = GITS_BASER_TYPE(val);
26053faf24eaSShanker Donthineni bool indirect = false;
2606d5df9dc9SMarc Zyngier u32 order;
26071ac19ca6SMarc Zyngier
2608d5df9dc9SMarc Zyngier if (type == GITS_BASER_TYPE_NONE)
26091ac19ca6SMarc Zyngier continue;
26101ac19ca6SMarc Zyngier
2611d5df9dc9SMarc Zyngier if (its_probe_baser_psz(its, baser)) {
2612d5df9dc9SMarc Zyngier its_free_tables(its);
2613d5df9dc9SMarc Zyngier return -ENXIO;
2614d5df9dc9SMarc Zyngier }
2615d5df9dc9SMarc Zyngier
2616d5df9dc9SMarc Zyngier order = get_order(baser->psz);
2617d5df9dc9SMarc Zyngier
2618d5df9dc9SMarc Zyngier switch (type) {
26194cacac57SMarc Zyngier case GITS_BASER_TYPE_DEVICE:
2620d5df9dc9SMarc Zyngier indirect = its_parse_indirect_baser(its, baser, &order,
2621576a8342SMarc Zyngier device_ids(its));
26228d565748SZenghui Yu break;
26238d565748SZenghui Yu
26244cacac57SMarc Zyngier case GITS_BASER_TYPE_VCPU:
26255e516846SMarc Zyngier if (is_v4_1(its)) {
26265e516846SMarc Zyngier struct its_node *sibling;
26275e516846SMarc Zyngier
26285e516846SMarc Zyngier WARN_ON(i != 2);
26295e516846SMarc Zyngier if ((sibling = find_sibling_its(its))) {
26305e516846SMarc Zyngier *baser = sibling->tables[2];
26315e516846SMarc Zyngier its_write_baser(its, baser, baser->val);
26325e516846SMarc Zyngier continue;
26335e516846SMarc Zyngier }
26345e516846SMarc Zyngier }
26355e516846SMarc Zyngier
2636d5df9dc9SMarc Zyngier indirect = its_parse_indirect_baser(its, baser, &order,
263732bd44dcSShanker Donthineni ITS_MAX_VPEID_BITS);
26384cacac57SMarc Zyngier break;
26394cacac57SMarc Zyngier }
2640f54b97edSMarc Zyngier
2641d5df9dc9SMarc Zyngier err = its_setup_baser(its, baser, cache, shr, order, indirect);
26429347359aSShanker Donthineni if (err < 0) {
26439347359aSShanker Donthineni its_free_tables(its);
26449347359aSShanker Donthineni return err;
264530f21363SRobert Richter }
264630f21363SRobert Richter
26479347359aSShanker Donthineni /* Update settings which will be used for next BASERn */
26489347359aSShanker Donthineni cache = baser->val & GITS_BASER_CACHEABILITY_MASK;
26499347359aSShanker Donthineni shr = baser->val & GITS_BASER_SHAREABILITY_MASK;
26501ac19ca6SMarc Zyngier }
26511ac19ca6SMarc Zyngier
26521ac19ca6SMarc Zyngier return 0;
26531ac19ca6SMarc Zyngier }
26541ac19ca6SMarc Zyngier
inherit_vpe_l1_table_from_its(void)26555e516846SMarc Zyngier static u64 inherit_vpe_l1_table_from_its(void)
26565e516846SMarc Zyngier {
26575e516846SMarc Zyngier struct its_node *its;
26585e516846SMarc Zyngier u64 val;
26595e516846SMarc Zyngier u32 aff;
26605e516846SMarc Zyngier
26615e516846SMarc Zyngier val = gic_read_typer(gic_data_rdist_rd_base() + GICR_TYPER);
26625e516846SMarc Zyngier aff = compute_common_aff(val);
26635e516846SMarc Zyngier
26645e516846SMarc Zyngier list_for_each_entry(its, &its_nodes, entry) {
26655e516846SMarc Zyngier u64 baser, addr;
26665e516846SMarc Zyngier
26675e516846SMarc Zyngier if (!is_v4_1(its))
26685e516846SMarc Zyngier continue;
26695e516846SMarc Zyngier
26705e516846SMarc Zyngier if (!FIELD_GET(GITS_TYPER_SVPET, its->typer))
26715e516846SMarc Zyngier continue;
26725e516846SMarc Zyngier
26735e516846SMarc Zyngier if (aff != compute_its_aff(its))
26745e516846SMarc Zyngier continue;
26755e516846SMarc Zyngier
26765e516846SMarc Zyngier /* GICv4.1 guarantees that the vPE table is GITS_BASER2 */
26775e516846SMarc Zyngier baser = its->tables[2].val;
26785e516846SMarc Zyngier if (!(baser & GITS_BASER_VALID))
26795e516846SMarc Zyngier continue;
26805e516846SMarc Zyngier
26815e516846SMarc Zyngier /* We have a winner! */
26828b718d40SZenghui Yu gic_data_rdist()->vpe_l1_base = its->tables[2].base;
26838b718d40SZenghui Yu
26845e516846SMarc Zyngier val = GICR_VPROPBASER_4_1_VALID;
26855e516846SMarc Zyngier if (baser & GITS_BASER_INDIRECT)
26865e516846SMarc Zyngier val |= GICR_VPROPBASER_4_1_INDIRECT;
26875e516846SMarc Zyngier val |= FIELD_PREP(GICR_VPROPBASER_4_1_PAGE_SIZE,
26885e516846SMarc Zyngier FIELD_GET(GITS_BASER_PAGE_SIZE_MASK, baser));
26895e516846SMarc Zyngier switch (FIELD_GET(GITS_BASER_PAGE_SIZE_MASK, baser)) {
26905e516846SMarc Zyngier case GIC_PAGE_SIZE_64K:
26915e516846SMarc Zyngier addr = GITS_BASER_ADDR_48_to_52(baser);
26925e516846SMarc Zyngier break;
26935e516846SMarc Zyngier default:
26945e516846SMarc Zyngier addr = baser & GENMASK_ULL(47, 12);
26955e516846SMarc Zyngier break;
26965e516846SMarc Zyngier }
26975e516846SMarc Zyngier val |= FIELD_PREP(GICR_VPROPBASER_4_1_ADDR, addr >> 12);
2698f7e84c8eSMarc Zyngier if (rdists_support_shareable()) {
26995e516846SMarc Zyngier val |= FIELD_PREP(GICR_VPROPBASER_SHAREABILITY_MASK,
27005e516846SMarc Zyngier FIELD_GET(GITS_BASER_SHAREABILITY_MASK, baser));
27015e516846SMarc Zyngier val |= FIELD_PREP(GICR_VPROPBASER_INNER_CACHEABILITY_MASK,
27025e516846SMarc Zyngier FIELD_GET(GITS_BASER_INNER_CACHEABILITY_MASK, baser));
2703f7e84c8eSMarc Zyngier }
27045e516846SMarc Zyngier val |= FIELD_PREP(GICR_VPROPBASER_4_1_SIZE, GITS_BASER_NR_PAGES(baser) - 1);
27055e516846SMarc Zyngier
27065e516846SMarc Zyngier return val;
27075e516846SMarc Zyngier }
27085e516846SMarc Zyngier
27095e516846SMarc Zyngier return 0;
27105e516846SMarc Zyngier }
27115e516846SMarc Zyngier
inherit_vpe_l1_table_from_rd(cpumask_t ** mask)27125e516846SMarc Zyngier static u64 inherit_vpe_l1_table_from_rd(cpumask_t **mask)
27135e516846SMarc Zyngier {
27145e516846SMarc Zyngier u32 aff;
27155e516846SMarc Zyngier u64 val;
27165e516846SMarc Zyngier int cpu;
27175e516846SMarc Zyngier
27185e516846SMarc Zyngier val = gic_read_typer(gic_data_rdist_rd_base() + GICR_TYPER);
27195e516846SMarc Zyngier aff = compute_common_aff(val);
27205e516846SMarc Zyngier
27215e516846SMarc Zyngier for_each_possible_cpu(cpu) {
27225e516846SMarc Zyngier void __iomem *base = gic_data_rdist_cpu(cpu)->rd_base;
27235e516846SMarc Zyngier
27245e516846SMarc Zyngier if (!base || cpu == smp_processor_id())
27255e516846SMarc Zyngier continue;
27265e516846SMarc Zyngier
27275e516846SMarc Zyngier val = gic_read_typer(base + GICR_TYPER);
27284bccf1d7SZenghui Yu if (aff != compute_common_aff(val))
27295e516846SMarc Zyngier continue;
27305e516846SMarc Zyngier
27315e516846SMarc Zyngier /*
27325e516846SMarc Zyngier * At this point, we have a victim. This particular CPU
27335e516846SMarc Zyngier * has already booted, and has an affinity that matches
27345e516846SMarc Zyngier * ours wrt CommonLPIAff. Let's use its own VPROPBASER.
27355e516846SMarc Zyngier * Make sure we don't write the Z bit in that case.
27365e516846SMarc Zyngier */
27375186a6ccSZenghui Yu val = gicr_read_vpropbaser(base + SZ_128K + GICR_VPROPBASER);
27385e516846SMarc Zyngier val &= ~GICR_VPROPBASER_4_1_Z;
27395e516846SMarc Zyngier
27408b718d40SZenghui Yu gic_data_rdist()->vpe_l1_base = gic_data_rdist_cpu(cpu)->vpe_l1_base;
27415e516846SMarc Zyngier *mask = gic_data_rdist_cpu(cpu)->vpe_table_mask;
27425e516846SMarc Zyngier
27435e516846SMarc Zyngier return val;
27445e516846SMarc Zyngier }
27455e516846SMarc Zyngier
27465e516846SMarc Zyngier return 0;
27475e516846SMarc Zyngier }
27485e516846SMarc Zyngier
allocate_vpe_l2_table(int cpu,u32 id)27494e6437f1SZenghui Yu static bool allocate_vpe_l2_table(int cpu, u32 id)
27504e6437f1SZenghui Yu {
27514e6437f1SZenghui Yu void __iomem *base = gic_data_rdist_cpu(cpu)->rd_base;
2752490d332eSMarc Zyngier unsigned int psz, esz, idx, npg, gpsz;
2753490d332eSMarc Zyngier u64 val;
27544e6437f1SZenghui Yu struct page *page;
27554e6437f1SZenghui Yu __le64 *table;
27564e6437f1SZenghui Yu
27574e6437f1SZenghui Yu if (!gic_rdists->has_rvpeid)
27584e6437f1SZenghui Yu return true;
27594e6437f1SZenghui Yu
276028d160deSMarc Zyngier /* Skip non-present CPUs */
276128d160deSMarc Zyngier if (!base)
276228d160deSMarc Zyngier return true;
276328d160deSMarc Zyngier
27645186a6ccSZenghui Yu val = gicr_read_vpropbaser(base + SZ_128K + GICR_VPROPBASER);
27654e6437f1SZenghui Yu
27664e6437f1SZenghui Yu esz = FIELD_GET(GICR_VPROPBASER_4_1_ENTRY_SIZE, val) + 1;
27674e6437f1SZenghui Yu gpsz = FIELD_GET(GICR_VPROPBASER_4_1_PAGE_SIZE, val);
27684e6437f1SZenghui Yu npg = FIELD_GET(GICR_VPROPBASER_4_1_SIZE, val) + 1;
27694e6437f1SZenghui Yu
27704e6437f1SZenghui Yu switch (gpsz) {
27714e6437f1SZenghui Yu default:
27724e6437f1SZenghui Yu WARN_ON(1);
2773df561f66SGustavo A. R. Silva fallthrough;
27744e6437f1SZenghui Yu case GIC_PAGE_SIZE_4K:
27754e6437f1SZenghui Yu psz = SZ_4K;
27764e6437f1SZenghui Yu break;
27774e6437f1SZenghui Yu case GIC_PAGE_SIZE_16K:
27784e6437f1SZenghui Yu psz = SZ_16K;
27794e6437f1SZenghui Yu break;
27804e6437f1SZenghui Yu case GIC_PAGE_SIZE_64K:
27814e6437f1SZenghui Yu psz = SZ_64K;
27824e6437f1SZenghui Yu break;
27834e6437f1SZenghui Yu }
27844e6437f1SZenghui Yu
27854e6437f1SZenghui Yu /* Don't allow vpe_id that exceeds single, flat table limit */
27864e6437f1SZenghui Yu if (!(val & GICR_VPROPBASER_4_1_INDIRECT))
27874e6437f1SZenghui Yu return (id < (npg * psz / (esz * SZ_8)));
27884e6437f1SZenghui Yu
27894e6437f1SZenghui Yu /* Compute 1st level table index & check if that exceeds table limit */
27904e6437f1SZenghui Yu idx = id >> ilog2(psz / (esz * SZ_8));
27914e6437f1SZenghui Yu if (idx >= (npg * psz / GITS_LVL1_ENTRY_SIZE))
27924e6437f1SZenghui Yu return false;
27934e6437f1SZenghui Yu
27944e6437f1SZenghui Yu table = gic_data_rdist_cpu(cpu)->vpe_l1_base;
27954e6437f1SZenghui Yu
27964e6437f1SZenghui Yu /* Allocate memory for 2nd level table */
27974e6437f1SZenghui Yu if (!table[idx]) {
27984e6437f1SZenghui Yu page = alloc_pages(GFP_KERNEL | __GFP_ZERO, get_order(psz));
27994e6437f1SZenghui Yu if (!page)
28004e6437f1SZenghui Yu return false;
28014e6437f1SZenghui Yu
28024e6437f1SZenghui Yu /* Flush Lvl2 table to PoC if hw doesn't support coherency */
28034e6437f1SZenghui Yu if (!(val & GICR_VPROPBASER_SHAREABILITY_MASK))
28044e6437f1SZenghui Yu gic_flush_dcache_to_poc(page_address(page), psz);
28054e6437f1SZenghui Yu
28064e6437f1SZenghui Yu table[idx] = cpu_to_le64(page_to_phys(page) | GITS_BASER_VALID);
28074e6437f1SZenghui Yu
28084e6437f1SZenghui Yu /* Flush Lvl1 entry to PoC if hw doesn't support coherency */
28094e6437f1SZenghui Yu if (!(val & GICR_VPROPBASER_SHAREABILITY_MASK))
28104e6437f1SZenghui Yu gic_flush_dcache_to_poc(table + idx, GITS_LVL1_ENTRY_SIZE);
28114e6437f1SZenghui Yu
28124e6437f1SZenghui Yu /* Ensure updated table contents are visible to RD hardware */
28134e6437f1SZenghui Yu dsb(sy);
28144e6437f1SZenghui Yu }
28154e6437f1SZenghui Yu
28164e6437f1SZenghui Yu return true;
28174e6437f1SZenghui Yu }
28184e6437f1SZenghui Yu
allocate_vpe_l1_table(void)28195e516846SMarc Zyngier static int allocate_vpe_l1_table(void)
28205e516846SMarc Zyngier {
28215e516846SMarc Zyngier void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
28225e516846SMarc Zyngier u64 val, gpsz, npg, pa;
28235e516846SMarc Zyngier unsigned int psz = SZ_64K;
28245e516846SMarc Zyngier unsigned int np, epp, esz;
28255e516846SMarc Zyngier struct page *page;
28265e516846SMarc Zyngier
28275e516846SMarc Zyngier if (!gic_rdists->has_rvpeid)
28285e516846SMarc Zyngier return 0;
28295e516846SMarc Zyngier
28305e516846SMarc Zyngier /*
28315e516846SMarc Zyngier * if VPENDBASER.Valid is set, disable any previously programmed
28325e516846SMarc Zyngier * VPE by setting PendingLast while clearing Valid. This has the
28335e516846SMarc Zyngier * effect of making sure no doorbell will be generated and we can
28345e516846SMarc Zyngier * then safely clear VPROPBASER.Valid.
28355e516846SMarc Zyngier */
28365186a6ccSZenghui Yu if (gicr_read_vpendbaser(vlpi_base + GICR_VPENDBASER) & GICR_VPENDBASER_Valid)
28375186a6ccSZenghui Yu gicr_write_vpendbaser(GICR_VPENDBASER_PendingLast,
28385e516846SMarc Zyngier vlpi_base + GICR_VPENDBASER);
28395e516846SMarc Zyngier
28405e516846SMarc Zyngier /*
28415e516846SMarc Zyngier * If we can inherit the configuration from another RD, let's do
28425e516846SMarc Zyngier * so. Otherwise, we have to go through the allocation process. We
28435e516846SMarc Zyngier * assume that all RDs have the exact same requirements, as
28445e516846SMarc Zyngier * nothing will work otherwise.
28455e516846SMarc Zyngier */
28465e516846SMarc Zyngier val = inherit_vpe_l1_table_from_rd(&gic_data_rdist()->vpe_table_mask);
28475e516846SMarc Zyngier if (val & GICR_VPROPBASER_4_1_VALID)
28485e516846SMarc Zyngier goto out;
28495e516846SMarc Zyngier
2850d1bd7e0bSZenghui Yu gic_data_rdist()->vpe_table_mask = kzalloc(sizeof(cpumask_t), GFP_ATOMIC);
28515e516846SMarc Zyngier if (!gic_data_rdist()->vpe_table_mask)
28525e516846SMarc Zyngier return -ENOMEM;
28535e516846SMarc Zyngier
28545e516846SMarc Zyngier val = inherit_vpe_l1_table_from_its();
28555e516846SMarc Zyngier if (val & GICR_VPROPBASER_4_1_VALID)
28565e516846SMarc Zyngier goto out;
28575e516846SMarc Zyngier
28585e516846SMarc Zyngier /* First probe the page size */
28595e516846SMarc Zyngier val = FIELD_PREP(GICR_VPROPBASER_4_1_PAGE_SIZE, GIC_PAGE_SIZE_64K);
28605186a6ccSZenghui Yu gicr_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER);
28615186a6ccSZenghui Yu val = gicr_read_vpropbaser(vlpi_base + GICR_VPROPBASER);
28625e516846SMarc Zyngier gpsz = FIELD_GET(GICR_VPROPBASER_4_1_PAGE_SIZE, val);
28635e516846SMarc Zyngier esz = FIELD_GET(GICR_VPROPBASER_4_1_ENTRY_SIZE, val);
28645e516846SMarc Zyngier
28655e516846SMarc Zyngier switch (gpsz) {
28665e516846SMarc Zyngier default:
28675e516846SMarc Zyngier gpsz = GIC_PAGE_SIZE_4K;
2868df561f66SGustavo A. R. Silva fallthrough;
28695e516846SMarc Zyngier case GIC_PAGE_SIZE_4K:
28705e516846SMarc Zyngier psz = SZ_4K;
28715e516846SMarc Zyngier break;
28725e516846SMarc Zyngier case GIC_PAGE_SIZE_16K:
28735e516846SMarc Zyngier psz = SZ_16K;
28745e516846SMarc Zyngier break;
28755e516846SMarc Zyngier case GIC_PAGE_SIZE_64K:
28765e516846SMarc Zyngier psz = SZ_64K;
28775e516846SMarc Zyngier break;
28785e516846SMarc Zyngier }
28795e516846SMarc Zyngier
28805e516846SMarc Zyngier /*
28815e516846SMarc Zyngier * Start populating the register from scratch, including RO fields
28825e516846SMarc Zyngier * (which we want to print in debug cases...)
28835e516846SMarc Zyngier */
28845e516846SMarc Zyngier val = 0;
28855e516846SMarc Zyngier val |= FIELD_PREP(GICR_VPROPBASER_4_1_PAGE_SIZE, gpsz);
28865e516846SMarc Zyngier val |= FIELD_PREP(GICR_VPROPBASER_4_1_ENTRY_SIZE, esz);
28875e516846SMarc Zyngier
28885e516846SMarc Zyngier /* How many entries per GIC page? */
28895e516846SMarc Zyngier esz++;
28905e516846SMarc Zyngier epp = psz / (esz * SZ_8);
28915e516846SMarc Zyngier
28925e516846SMarc Zyngier /*
28935e516846SMarc Zyngier * If we need more than just a single L1 page, flag the table
28945e516846SMarc Zyngier * as indirect and compute the number of required L1 pages.
28955e516846SMarc Zyngier */
28965e516846SMarc Zyngier if (epp < ITS_MAX_VPEID) {
28975e516846SMarc Zyngier int nl2;
28985e516846SMarc Zyngier
28995e516846SMarc Zyngier val |= GICR_VPROPBASER_4_1_INDIRECT;
29005e516846SMarc Zyngier
29015e516846SMarc Zyngier /* Number of L2 pages required to cover the VPEID space */
29025e516846SMarc Zyngier nl2 = DIV_ROUND_UP(ITS_MAX_VPEID, epp);
29035e516846SMarc Zyngier
29045e516846SMarc Zyngier /* Number of L1 pages to point to the L2 pages */
29055e516846SMarc Zyngier npg = DIV_ROUND_UP(nl2 * SZ_8, psz);
29065e516846SMarc Zyngier } else {
29075e516846SMarc Zyngier npg = 1;
29085e516846SMarc Zyngier }
29095e516846SMarc Zyngier
2910e88bd316SZenghui Yu val |= FIELD_PREP(GICR_VPROPBASER_4_1_SIZE, npg - 1);
29115e516846SMarc Zyngier
29125e516846SMarc Zyngier /* Right, that's the number of CPU pages we need for L1 */
29135e516846SMarc Zyngier np = DIV_ROUND_UP(npg * psz, PAGE_SIZE);
29145e516846SMarc Zyngier
29155e516846SMarc Zyngier pr_debug("np = %d, npg = %lld, psz = %d, epp = %d, esz = %d\n",
29165e516846SMarc Zyngier np, npg, psz, epp, esz);
2917d1bd7e0bSZenghui Yu page = alloc_pages(GFP_ATOMIC | __GFP_ZERO, get_order(np * PAGE_SIZE));
29185e516846SMarc Zyngier if (!page)
29195e516846SMarc Zyngier return -ENOMEM;
29205e516846SMarc Zyngier
29218b718d40SZenghui Yu gic_data_rdist()->vpe_l1_base = page_address(page);
29225e516846SMarc Zyngier pa = virt_to_phys(page_address(page));
29235e516846SMarc Zyngier WARN_ON(!IS_ALIGNED(pa, psz));
29245e516846SMarc Zyngier
29255e516846SMarc Zyngier val |= FIELD_PREP(GICR_VPROPBASER_4_1_ADDR, pa >> 12);
2926f7e84c8eSMarc Zyngier if (rdists_support_shareable()) {
29275e516846SMarc Zyngier val |= GICR_VPROPBASER_RaWb;
29285e516846SMarc Zyngier val |= GICR_VPROPBASER_InnerShareable;
2929f7e84c8eSMarc Zyngier }
29305e516846SMarc Zyngier val |= GICR_VPROPBASER_4_1_Z;
29315e516846SMarc Zyngier val |= GICR_VPROPBASER_4_1_VALID;
29325e516846SMarc Zyngier
29335e516846SMarc Zyngier out:
29345186a6ccSZenghui Yu gicr_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER);
29355e516846SMarc Zyngier cpumask_set_cpu(smp_processor_id(), gic_data_rdist()->vpe_table_mask);
29365e516846SMarc Zyngier
29375e516846SMarc Zyngier pr_debug("CPU%d: VPROPBASER = %llx %*pbl\n",
29385e516846SMarc Zyngier smp_processor_id(), val,
29395e516846SMarc Zyngier cpumask_pr_args(gic_data_rdist()->vpe_table_mask));
29405e516846SMarc Zyngier
29415e516846SMarc Zyngier return 0;
29425e516846SMarc Zyngier }
29435e516846SMarc Zyngier
its_alloc_collections(struct its_node * its)29441ac19ca6SMarc Zyngier static int its_alloc_collections(struct its_node *its)
29451ac19ca6SMarc Zyngier {
294683559b47SMarc Zyngier int i;
294783559b47SMarc Zyngier
29486396bb22SKees Cook its->collections = kcalloc(nr_cpu_ids, sizeof(*its->collections),
29491ac19ca6SMarc Zyngier GFP_KERNEL);
29501ac19ca6SMarc Zyngier if (!its->collections)
29511ac19ca6SMarc Zyngier return -ENOMEM;
29521ac19ca6SMarc Zyngier
295383559b47SMarc Zyngier for (i = 0; i < nr_cpu_ids; i++)
295483559b47SMarc Zyngier its->collections[i].target_address = ~0ULL;
295583559b47SMarc Zyngier
29561ac19ca6SMarc Zyngier return 0;
29571ac19ca6SMarc Zyngier }
29581ac19ca6SMarc Zyngier
its_allocate_pending_table(gfp_t gfp_flags)29597c297a2dSMarc Zyngier static struct page *its_allocate_pending_table(gfp_t gfp_flags)
29607c297a2dSMarc Zyngier {
29617c297a2dSMarc Zyngier struct page *pend_page;
2962adaab500SMarc Zyngier
29637c297a2dSMarc Zyngier pend_page = alloc_pages(gfp_flags | __GFP_ZERO,
2964adaab500SMarc Zyngier get_order(LPI_PENDBASE_SZ));
29657c297a2dSMarc Zyngier if (!pend_page)
29667c297a2dSMarc Zyngier return NULL;
29677c297a2dSMarc Zyngier
29687c297a2dSMarc Zyngier /* Make sure the GIC will observe the zero-ed page */
29697c297a2dSMarc Zyngier gic_flush_dcache_to_poc(page_address(pend_page), LPI_PENDBASE_SZ);
29707c297a2dSMarc Zyngier
29717c297a2dSMarc Zyngier return pend_page;
29727c297a2dSMarc Zyngier }
29737c297a2dSMarc Zyngier
its_free_pending_table(struct page * pt)29747d75bbb4SMarc Zyngier static void its_free_pending_table(struct page *pt)
29757d75bbb4SMarc Zyngier {
2976adaab500SMarc Zyngier free_pages((unsigned long)page_address(pt), get_order(LPI_PENDBASE_SZ));
29777d75bbb4SMarc Zyngier }
29787d75bbb4SMarc Zyngier
2979c6e2ccb6SMarc Zyngier /*
29805e2c9f9aSMarc Zyngier * Booting with kdump and LPIs enabled is generally fine. Any other
29815e2c9f9aSMarc Zyngier * case is wrong in the absence of firmware/EFI support.
2982c6e2ccb6SMarc Zyngier */
enabled_lpis_allowed(void)2983c440a9d9SMarc Zyngier static bool enabled_lpis_allowed(void)
2984c440a9d9SMarc Zyngier {
29855e2c9f9aSMarc Zyngier phys_addr_t addr;
29865e2c9f9aSMarc Zyngier u64 val;
2987c6e2ccb6SMarc Zyngier
29885e2c9f9aSMarc Zyngier /* Check whether the property table is in a reserved region */
29895e2c9f9aSMarc Zyngier val = gicr_read_propbaser(gic_data_rdist_rd_base() + GICR_PROPBASER);
29905e2c9f9aSMarc Zyngier addr = val & GENMASK_ULL(51, 12);
29915e2c9f9aSMarc Zyngier
29925e2c9f9aSMarc Zyngier return gic_check_reserved_range(addr, LPI_PROPBASE_SZ);
2993c440a9d9SMarc Zyngier }
2994c440a9d9SMarc Zyngier
allocate_lpi_tables(void)299511e37d35SMarc Zyngier static int __init allocate_lpi_tables(void)
299611e37d35SMarc Zyngier {
2997c440a9d9SMarc Zyngier u64 val;
299811e37d35SMarc Zyngier int err, cpu;
299911e37d35SMarc Zyngier
3000c440a9d9SMarc Zyngier /*
3001c440a9d9SMarc Zyngier * If LPIs are enabled while we run this from the boot CPU,
3002c440a9d9SMarc Zyngier * flag the RD tables as pre-allocated if the stars do align.
3003c440a9d9SMarc Zyngier */
3004c440a9d9SMarc Zyngier val = readl_relaxed(gic_data_rdist_rd_base() + GICR_CTLR);
3005c440a9d9SMarc Zyngier if ((val & GICR_CTLR_ENABLE_LPIS) && enabled_lpis_allowed()) {
3006c440a9d9SMarc Zyngier gic_rdists->flags |= (RDIST_FLAGS_RD_TABLES_PREALLOCATED |
3007c440a9d9SMarc Zyngier RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING);
3008c440a9d9SMarc Zyngier pr_info("GICv3: Using preallocated redistributor tables\n");
3009c440a9d9SMarc Zyngier }
3010c440a9d9SMarc Zyngier
301111e37d35SMarc Zyngier err = its_setup_lpi_prop_table();
301211e37d35SMarc Zyngier if (err)
301311e37d35SMarc Zyngier return err;
301411e37d35SMarc Zyngier
301511e37d35SMarc Zyngier /*
301611e37d35SMarc Zyngier * We allocate all the pending tables anyway, as we may have a
301711e37d35SMarc Zyngier * mix of RDs that have had LPIs enabled, and some that
301811e37d35SMarc Zyngier * don't. We'll free the unused ones as each CPU comes online.
301911e37d35SMarc Zyngier */
302011e37d35SMarc Zyngier for_each_possible_cpu(cpu) {
302111e37d35SMarc Zyngier struct page *pend_page;
302211e37d35SMarc Zyngier
302311e37d35SMarc Zyngier pend_page = its_allocate_pending_table(GFP_NOWAIT);
302411e37d35SMarc Zyngier if (!pend_page) {
302511e37d35SMarc Zyngier pr_err("Failed to allocate PENDBASE for CPU%d\n", cpu);
302611e37d35SMarc Zyngier return -ENOMEM;
302711e37d35SMarc Zyngier }
302811e37d35SMarc Zyngier
302911e37d35SMarc Zyngier gic_data_rdist_cpu(cpu)->pend_page = pend_page;
303011e37d35SMarc Zyngier }
303111e37d35SMarc Zyngier
303211e37d35SMarc Zyngier return 0;
303311e37d35SMarc Zyngier }
303411e37d35SMarc Zyngier
read_vpend_dirty_clear(void __iomem * vlpi_base)3035af27e416SMarc Zyngier static u64 read_vpend_dirty_clear(void __iomem *vlpi_base)
30366479450fSHeyi Guo {
30376479450fSHeyi Guo u32 count = 1000000; /* 1s! */
30386479450fSHeyi Guo bool clean;
30396479450fSHeyi Guo u64 val;
30406479450fSHeyi Guo
30416479450fSHeyi Guo do {
30425186a6ccSZenghui Yu val = gicr_read_vpendbaser(vlpi_base + GICR_VPENDBASER);
30436479450fSHeyi Guo clean = !(val & GICR_VPENDBASER_Dirty);
30446479450fSHeyi Guo if (!clean) {
30456479450fSHeyi Guo count--;
30466479450fSHeyi Guo cpu_relax();
30476479450fSHeyi Guo udelay(1);
30486479450fSHeyi Guo }
30496479450fSHeyi Guo } while (!clean && count);
30506479450fSHeyi Guo
3051af27e416SMarc Zyngier if (unlikely(!clean))
3052e64fab1aSMarc Zyngier pr_err_ratelimited("ITS virtual pending table not cleaning\n");
3053af27e416SMarc Zyngier
3054af27e416SMarc Zyngier return val;
3055e64fab1aSMarc Zyngier }
3056e64fab1aSMarc Zyngier
its_clear_vpend_valid(void __iomem * vlpi_base,u64 clr,u64 set)3057af27e416SMarc Zyngier static u64 its_clear_vpend_valid(void __iomem *vlpi_base, u64 clr, u64 set)
3058af27e416SMarc Zyngier {
3059af27e416SMarc Zyngier u64 val;
3060af27e416SMarc Zyngier
3061af27e416SMarc Zyngier /* Make sure we wait until the RD is done with the initial scan */
3062af27e416SMarc Zyngier val = read_vpend_dirty_clear(vlpi_base);
3063af27e416SMarc Zyngier val &= ~GICR_VPENDBASER_Valid;
3064af27e416SMarc Zyngier val &= ~clr;
3065af27e416SMarc Zyngier val |= set;
3066af27e416SMarc Zyngier gicr_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER);
3067af27e416SMarc Zyngier
3068af27e416SMarc Zyngier val = read_vpend_dirty_clear(vlpi_base);
3069af27e416SMarc Zyngier if (unlikely(val & GICR_VPENDBASER_Dirty))
3070af27e416SMarc Zyngier val |= GICR_VPENDBASER_PendingLast;
3071af27e416SMarc Zyngier
30726479450fSHeyi Guo return val;
30736479450fSHeyi Guo }
30746479450fSHeyi Guo
its_cpu_init_lpis(void)30751ac19ca6SMarc Zyngier static void its_cpu_init_lpis(void)
30761ac19ca6SMarc Zyngier {
30771ac19ca6SMarc Zyngier void __iomem *rbase = gic_data_rdist_rd_base();
30781ac19ca6SMarc Zyngier struct page *pend_page;
307911e37d35SMarc Zyngier phys_addr_t paddr;
30801ac19ca6SMarc Zyngier u64 val, tmp;
30811ac19ca6SMarc Zyngier
3082c0cdc890SValentin Schneider if (gic_data_rdist()->flags & RD_LOCAL_LPI_ENABLED)
30831ac19ca6SMarc Zyngier return;
30841ac19ca6SMarc Zyngier
3085c440a9d9SMarc Zyngier val = readl_relaxed(rbase + GICR_CTLR);
3086c440a9d9SMarc Zyngier if ((gic_rdists->flags & RDIST_FLAGS_RD_TABLES_PREALLOCATED) &&
3087c440a9d9SMarc Zyngier (val & GICR_CTLR_ENABLE_LPIS)) {
3088f842ca8eSMarc Zyngier /*
3089f842ca8eSMarc Zyngier * Check that we get the same property table on all
3090f842ca8eSMarc Zyngier * RDs. If we don't, this is hopeless.
3091f842ca8eSMarc Zyngier */
3092f842ca8eSMarc Zyngier paddr = gicr_read_propbaser(rbase + GICR_PROPBASER);
3093f842ca8eSMarc Zyngier paddr &= GENMASK_ULL(51, 12);
3094f842ca8eSMarc Zyngier if (WARN_ON(gic_rdists->prop_table_pa != paddr))
3095f842ca8eSMarc Zyngier add_taint(TAINT_CRAP, LOCKDEP_STILL_OK);
3096f842ca8eSMarc Zyngier
3097c440a9d9SMarc Zyngier paddr = gicr_read_pendbaser(rbase + GICR_PENDBASER);
3098c440a9d9SMarc Zyngier paddr &= GENMASK_ULL(51, 16);
3099c440a9d9SMarc Zyngier
31005e2c9f9aSMarc Zyngier WARN_ON(!gic_check_reserved_range(paddr, LPI_PENDBASE_SZ));
3101d23bc2bcSValentin Schneider gic_data_rdist()->flags |= RD_LOCAL_PENDTABLE_PREALLOCATED;
3102c440a9d9SMarc Zyngier
3103c440a9d9SMarc Zyngier goto out;
3104c440a9d9SMarc Zyngier }
3105c440a9d9SMarc Zyngier
310611e37d35SMarc Zyngier pend_page = gic_data_rdist()->pend_page;
31071ac19ca6SMarc Zyngier paddr = page_to_phys(pend_page);
31081ac19ca6SMarc Zyngier
31091ac19ca6SMarc Zyngier /* set PROPBASE */
3110e1a2e201SMarc Zyngier val = (gic_rdists->prop_table_pa |
31111ac19ca6SMarc Zyngier GICR_PROPBASER_InnerShareable |
31122fd632a0SShanker Donthineni GICR_PROPBASER_RaWaWb |
31131ac19ca6SMarc Zyngier ((LPI_NRBITS - 1) & GICR_PROPBASER_IDBITS_MASK));
31141ac19ca6SMarc Zyngier
31150968a619SVladimir Murzin gicr_write_propbaser(val, rbase + GICR_PROPBASER);
31160968a619SVladimir Murzin tmp = gicr_read_propbaser(rbase + GICR_PROPBASER);
31171ac19ca6SMarc Zyngier
3118f7e84c8eSMarc Zyngier if (!rdists_support_shareable())
3119a8707f55SSebastian Reichel tmp &= ~GICR_PROPBASER_SHAREABILITY_MASK;
3120a8707f55SSebastian Reichel
31211ac19ca6SMarc Zyngier if ((tmp ^ val) & GICR_PROPBASER_SHAREABILITY_MASK) {
3122241a386cSMarc Zyngier if (!(tmp & GICR_PROPBASER_SHAREABILITY_MASK)) {
3123241a386cSMarc Zyngier /*
3124241a386cSMarc Zyngier * The HW reports non-shareable, we must
3125241a386cSMarc Zyngier * remove the cacheability attributes as
3126241a386cSMarc Zyngier * well.
3127241a386cSMarc Zyngier */
3128241a386cSMarc Zyngier val &= ~(GICR_PROPBASER_SHAREABILITY_MASK |
3129241a386cSMarc Zyngier GICR_PROPBASER_CACHEABILITY_MASK);
3130241a386cSMarc Zyngier val |= GICR_PROPBASER_nC;
31310968a619SVladimir Murzin gicr_write_propbaser(val, rbase + GICR_PROPBASER);
3132241a386cSMarc Zyngier }
31331ac19ca6SMarc Zyngier pr_info_once("GIC: using cache flushing for LPI property table\n");
31341ac19ca6SMarc Zyngier gic_rdists->flags |= RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING;
31351ac19ca6SMarc Zyngier }
31361ac19ca6SMarc Zyngier
31371ac19ca6SMarc Zyngier /* set PENDBASE */
31381ac19ca6SMarc Zyngier val = (page_to_phys(pend_page) |
31394ad3e363SMarc Zyngier GICR_PENDBASER_InnerShareable |
31402fd632a0SShanker Donthineni GICR_PENDBASER_RaWaWb);
31411ac19ca6SMarc Zyngier
31420968a619SVladimir Murzin gicr_write_pendbaser(val, rbase + GICR_PENDBASER);
31430968a619SVladimir Murzin tmp = gicr_read_pendbaser(rbase + GICR_PENDBASER);
3144241a386cSMarc Zyngier
3145f7e84c8eSMarc Zyngier if (!rdists_support_shareable())
3146a8707f55SSebastian Reichel tmp &= ~GICR_PENDBASER_SHAREABILITY_MASK;
3147a8707f55SSebastian Reichel
3148241a386cSMarc Zyngier if (!(tmp & GICR_PENDBASER_SHAREABILITY_MASK)) {
3149241a386cSMarc Zyngier /*
3150241a386cSMarc Zyngier * The HW reports non-shareable, we must remove the
3151241a386cSMarc Zyngier * cacheability attributes as well.
3152241a386cSMarc Zyngier */
3153241a386cSMarc Zyngier val &= ~(GICR_PENDBASER_SHAREABILITY_MASK |
3154241a386cSMarc Zyngier GICR_PENDBASER_CACHEABILITY_MASK);
3155241a386cSMarc Zyngier val |= GICR_PENDBASER_nC;
31560968a619SVladimir Murzin gicr_write_pendbaser(val, rbase + GICR_PENDBASER);
3157241a386cSMarc Zyngier }
31581ac19ca6SMarc Zyngier
31591ac19ca6SMarc Zyngier /* Enable LPIs */
31601ac19ca6SMarc Zyngier val = readl_relaxed(rbase + GICR_CTLR);
31611ac19ca6SMarc Zyngier val |= GICR_CTLR_ENABLE_LPIS;
31621ac19ca6SMarc Zyngier writel_relaxed(val, rbase + GICR_CTLR);
31631ac19ca6SMarc Zyngier
31644332f541SOliver Upton out:
31655e516846SMarc Zyngier if (gic_rdists->has_vlpis && !gic_rdists->has_rvpeid) {
31666479450fSHeyi Guo void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
31676479450fSHeyi Guo
31686479450fSHeyi Guo /*
31696479450fSHeyi Guo * It's possible for CPU to receive VLPIs before it is
3170a359f757SIngo Molnar * scheduled as a vPE, especially for the first CPU, and the
31716479450fSHeyi Guo * VLPI with INTID larger than 2^(IDbits+1) will be considered
31726479450fSHeyi Guo * as out of range and dropped by GIC.
31736479450fSHeyi Guo * So we initialize IDbits to known value to avoid VLPI drop.
31746479450fSHeyi Guo */
31756479450fSHeyi Guo val = (LPI_NRBITS - 1) & GICR_VPROPBASER_IDBITS_MASK;
31766479450fSHeyi Guo pr_debug("GICv4: CPU%d: Init IDbits to 0x%llx for GICR_VPROPBASER\n",
31776479450fSHeyi Guo smp_processor_id(), val);
31785186a6ccSZenghui Yu gicr_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER);
31796479450fSHeyi Guo
31806479450fSHeyi Guo /*
31816479450fSHeyi Guo * Also clear Valid bit of GICR_VPENDBASER, in case some
31826479450fSHeyi Guo * ancient programming gets left in and has possibility of
31836479450fSHeyi Guo * corrupting memory.
31846479450fSHeyi Guo */
3185e64fab1aSMarc Zyngier val = its_clear_vpend_valid(vlpi_base, 0, 0);
31866479450fSHeyi Guo }
31876479450fSHeyi Guo
31885e516846SMarc Zyngier if (allocate_vpe_l1_table()) {
31895e516846SMarc Zyngier /*
31905e516846SMarc Zyngier * If the allocation has failed, we're in massive trouble.
31915e516846SMarc Zyngier * Disable direct injection, and pray that no VM was
31925e516846SMarc Zyngier * already running...
31935e516846SMarc Zyngier */
31945e516846SMarc Zyngier gic_rdists->has_rvpeid = false;
31955e516846SMarc Zyngier gic_rdists->has_vlpis = false;
31965e516846SMarc Zyngier }
31975e516846SMarc Zyngier
31981ac19ca6SMarc Zyngier /* Make sure the GIC has seen the above */
31991ac19ca6SMarc Zyngier dsb(sy);
3200c0cdc890SValentin Schneider gic_data_rdist()->flags |= RD_LOCAL_LPI_ENABLED;
3201c440a9d9SMarc Zyngier pr_info("GICv3: CPU%d: using %s LPI pending table @%pa\n",
320211e37d35SMarc Zyngier smp_processor_id(),
3203d23bc2bcSValentin Schneider gic_data_rdist()->flags & RD_LOCAL_PENDTABLE_PREALLOCATED ?
3204d23bc2bcSValentin Schneider "reserved" : "allocated",
320511e37d35SMarc Zyngier &paddr);
32061ac19ca6SMarc Zyngier }
32071ac19ca6SMarc Zyngier
its_cpu_init_collection(struct its_node * its)3208920181ceSDerek Basehore static void its_cpu_init_collection(struct its_node *its)
32091ac19ca6SMarc Zyngier {
3210920181ceSDerek Basehore int cpu = smp_processor_id();
32111ac19ca6SMarc Zyngier u64 target;
32121ac19ca6SMarc Zyngier
3213fbf8f40eSGanapatrao Kulkarni /* avoid cross node collections and its mapping */
3214fbf8f40eSGanapatrao Kulkarni if (its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144) {
3215fbf8f40eSGanapatrao Kulkarni struct device_node *cpu_node;
3216fbf8f40eSGanapatrao Kulkarni
3217fbf8f40eSGanapatrao Kulkarni cpu_node = of_get_cpu_node(cpu, NULL);
3218fbf8f40eSGanapatrao Kulkarni if (its->numa_node != NUMA_NO_NODE &&
3219fbf8f40eSGanapatrao Kulkarni its->numa_node != of_node_to_nid(cpu_node))
3220920181ceSDerek Basehore return;
3221fbf8f40eSGanapatrao Kulkarni }
3222fbf8f40eSGanapatrao Kulkarni
32231ac19ca6SMarc Zyngier /*
32241ac19ca6SMarc Zyngier * We now have to bind each collection to its target
32251ac19ca6SMarc Zyngier * redistributor.
32261ac19ca6SMarc Zyngier */
3227589ce5f4SMarc Zyngier if (gic_read_typer(its->base + GITS_TYPER) & GITS_TYPER_PTA) {
32281ac19ca6SMarc Zyngier /*
32291ac19ca6SMarc Zyngier * This ITS wants the physical address of the
32301ac19ca6SMarc Zyngier * redistributor.
32311ac19ca6SMarc Zyngier */
32321ac19ca6SMarc Zyngier target = gic_data_rdist()->phys_base;
32331ac19ca6SMarc Zyngier } else {
3234920181ceSDerek Basehore /* This ITS wants a linear CPU number. */
3235589ce5f4SMarc Zyngier target = gic_read_typer(gic_data_rdist_rd_base() + GICR_TYPER);
3236263fcd31SMarc Zyngier target = GICR_TYPER_CPU_NUMBER(target) << 16;
32371ac19ca6SMarc Zyngier }
32381ac19ca6SMarc Zyngier
32391ac19ca6SMarc Zyngier /* Perform collection mapping */
32401ac19ca6SMarc Zyngier its->collections[cpu].target_address = target;
32411ac19ca6SMarc Zyngier its->collections[cpu].col_id = cpu;
32421ac19ca6SMarc Zyngier
32431ac19ca6SMarc Zyngier its_send_mapc(its, &its->collections[cpu], 1);
32441ac19ca6SMarc Zyngier its_send_invall(its, &its->collections[cpu]);
32451ac19ca6SMarc Zyngier }
32461ac19ca6SMarc Zyngier
its_cpu_init_collections(void)3247920181ceSDerek Basehore static void its_cpu_init_collections(void)
3248920181ceSDerek Basehore {
3249920181ceSDerek Basehore struct its_node *its;
3250920181ceSDerek Basehore
3251a8db7456SSebastian Andrzej Siewior raw_spin_lock(&its_lock);
3252920181ceSDerek Basehore
3253920181ceSDerek Basehore list_for_each_entry(its, &its_nodes, entry)
3254920181ceSDerek Basehore its_cpu_init_collection(its);
3255920181ceSDerek Basehore
3256a8db7456SSebastian Andrzej Siewior raw_spin_unlock(&its_lock);
32571ac19ca6SMarc Zyngier }
325884a6a2e7SMarc Zyngier
its_find_device(struct its_node * its,u32 dev_id)325984a6a2e7SMarc Zyngier static struct its_device *its_find_device(struct its_node *its, u32 dev_id)
326084a6a2e7SMarc Zyngier {
326184a6a2e7SMarc Zyngier struct its_device *its_dev = NULL, *tmp;
32623e39e8f5SMarc Zyngier unsigned long flags;
326384a6a2e7SMarc Zyngier
32643e39e8f5SMarc Zyngier raw_spin_lock_irqsave(&its->lock, flags);
326584a6a2e7SMarc Zyngier
326684a6a2e7SMarc Zyngier list_for_each_entry(tmp, &its->its_device_list, entry) {
326784a6a2e7SMarc Zyngier if (tmp->device_id == dev_id) {
326884a6a2e7SMarc Zyngier its_dev = tmp;
326984a6a2e7SMarc Zyngier break;
327084a6a2e7SMarc Zyngier }
327184a6a2e7SMarc Zyngier }
327284a6a2e7SMarc Zyngier
32733e39e8f5SMarc Zyngier raw_spin_unlock_irqrestore(&its->lock, flags);
327484a6a2e7SMarc Zyngier
327584a6a2e7SMarc Zyngier return its_dev;
327684a6a2e7SMarc Zyngier }
327784a6a2e7SMarc Zyngier
its_get_baser(struct its_node * its,u32 type)3278466b7d16SShanker Donthineni static struct its_baser *its_get_baser(struct its_node *its, u32 type)
3279466b7d16SShanker Donthineni {
3280466b7d16SShanker Donthineni int i;
3281466b7d16SShanker Donthineni
3282466b7d16SShanker Donthineni for (i = 0; i < GITS_BASER_NR_REGS; i++) {
3283466b7d16SShanker Donthineni if (GITS_BASER_TYPE(its->tables[i].val) == type)
3284466b7d16SShanker Donthineni return &its->tables[i];
3285466b7d16SShanker Donthineni }
3286466b7d16SShanker Donthineni
3287466b7d16SShanker Donthineni return NULL;
3288466b7d16SShanker Donthineni }
3289466b7d16SShanker Donthineni
its_alloc_table_entry(struct its_node * its,struct its_baser * baser,u32 id)3290539d3782SShanker Donthineni static bool its_alloc_table_entry(struct its_node *its,
3291539d3782SShanker Donthineni struct its_baser *baser, u32 id)
32923faf24eaSShanker Donthineni {
32933faf24eaSShanker Donthineni struct page *page;
32943faf24eaSShanker Donthineni u32 esz, idx;
32953faf24eaSShanker Donthineni __le64 *table;
32963faf24eaSShanker Donthineni
32973faf24eaSShanker Donthineni /* Don't allow device id that exceeds single, flat table limit */
32983faf24eaSShanker Donthineni esz = GITS_BASER_ENTRY_SIZE(baser->val);
32993faf24eaSShanker Donthineni if (!(baser->val & GITS_BASER_INDIRECT))
330070cc81edSMarc Zyngier return (id < (PAGE_ORDER_TO_SIZE(baser->order) / esz));
33013faf24eaSShanker Donthineni
33023faf24eaSShanker Donthineni /* Compute 1st level table index & check if that exceeds table limit */
330370cc81edSMarc Zyngier idx = id >> ilog2(baser->psz / esz);
33043faf24eaSShanker Donthineni if (idx >= (PAGE_ORDER_TO_SIZE(baser->order) / GITS_LVL1_ENTRY_SIZE))
33053faf24eaSShanker Donthineni return false;
33063faf24eaSShanker Donthineni
33073faf24eaSShanker Donthineni table = baser->base;
33083faf24eaSShanker Donthineni
33093faf24eaSShanker Donthineni /* Allocate memory for 2nd level table */
33103faf24eaSShanker Donthineni if (!table[idx]) {
3311539d3782SShanker Donthineni page = alloc_pages_node(its->numa_node, GFP_KERNEL | __GFP_ZERO,
3312539d3782SShanker Donthineni get_order(baser->psz));
33133faf24eaSShanker Donthineni if (!page)
33143faf24eaSShanker Donthineni return false;
33153faf24eaSShanker Donthineni
33163faf24eaSShanker Donthineni /* Flush Lvl2 table to PoC if hw doesn't support coherency */
33173faf24eaSShanker Donthineni if (!(baser->val & GITS_BASER_SHAREABILITY_MASK))
3318328191c0SVladimir Murzin gic_flush_dcache_to_poc(page_address(page), baser->psz);
33193faf24eaSShanker Donthineni
33203faf24eaSShanker Donthineni table[idx] = cpu_to_le64(page_to_phys(page) | GITS_BASER_VALID);
33213faf24eaSShanker Donthineni
33223faf24eaSShanker Donthineni /* Flush Lvl1 entry to PoC if hw doesn't support coherency */
33233faf24eaSShanker Donthineni if (!(baser->val & GITS_BASER_SHAREABILITY_MASK))
3324328191c0SVladimir Murzin gic_flush_dcache_to_poc(table + idx, GITS_LVL1_ENTRY_SIZE);
33253faf24eaSShanker Donthineni
33263faf24eaSShanker Donthineni /* Ensure updated table contents are visible to ITS hardware */
33273faf24eaSShanker Donthineni dsb(sy);
33283faf24eaSShanker Donthineni }
33293faf24eaSShanker Donthineni
33303faf24eaSShanker Donthineni return true;
33313faf24eaSShanker Donthineni }
33323faf24eaSShanker Donthineni
its_alloc_device_table(struct its_node * its,u32 dev_id)333370cc81edSMarc Zyngier static bool its_alloc_device_table(struct its_node *its, u32 dev_id)
333470cc81edSMarc Zyngier {
333570cc81edSMarc Zyngier struct its_baser *baser;
333670cc81edSMarc Zyngier
333770cc81edSMarc Zyngier baser = its_get_baser(its, GITS_BASER_TYPE_DEVICE);
333870cc81edSMarc Zyngier
333970cc81edSMarc Zyngier /* Don't allow device id that exceeds ITS hardware limit */
334070cc81edSMarc Zyngier if (!baser)
3341576a8342SMarc Zyngier return (ilog2(dev_id) < device_ids(its));
334270cc81edSMarc Zyngier
3343539d3782SShanker Donthineni return its_alloc_table_entry(its, baser, dev_id);
334470cc81edSMarc Zyngier }
334570cc81edSMarc Zyngier
its_alloc_vpe_table(u32 vpe_id)33467d75bbb4SMarc Zyngier static bool its_alloc_vpe_table(u32 vpe_id)
33477d75bbb4SMarc Zyngier {
33487d75bbb4SMarc Zyngier struct its_node *its;
33494e6437f1SZenghui Yu int cpu;
33507d75bbb4SMarc Zyngier
33517d75bbb4SMarc Zyngier /*
33527d75bbb4SMarc Zyngier * Make sure the L2 tables are allocated on *all* v4 ITSs. We
33537d75bbb4SMarc Zyngier * could try and only do it on ITSs corresponding to devices
33547d75bbb4SMarc Zyngier * that have interrupts targeted at this VPE, but the
33557d75bbb4SMarc Zyngier * complexity becomes crazy (and you have tons of memory
33567d75bbb4SMarc Zyngier * anyway, right?).
33577d75bbb4SMarc Zyngier */
33587d75bbb4SMarc Zyngier list_for_each_entry(its, &its_nodes, entry) {
33597d75bbb4SMarc Zyngier struct its_baser *baser;
33607d75bbb4SMarc Zyngier
33610dd57fedSMarc Zyngier if (!is_v4(its))
33627d75bbb4SMarc Zyngier continue;
33637d75bbb4SMarc Zyngier
33647d75bbb4SMarc Zyngier baser = its_get_baser(its, GITS_BASER_TYPE_VCPU);
33657d75bbb4SMarc Zyngier if (!baser)
33667d75bbb4SMarc Zyngier return false;
33677d75bbb4SMarc Zyngier
3368539d3782SShanker Donthineni if (!its_alloc_table_entry(its, baser, vpe_id))
33697d75bbb4SMarc Zyngier return false;
33707d75bbb4SMarc Zyngier }
33717d75bbb4SMarc Zyngier
33724e6437f1SZenghui Yu /* Non v4.1? No need to iterate RDs and go back early. */
33734e6437f1SZenghui Yu if (!gic_rdists->has_rvpeid)
33744e6437f1SZenghui Yu return true;
33754e6437f1SZenghui Yu
33764e6437f1SZenghui Yu /*
33774e6437f1SZenghui Yu * Make sure the L2 tables are allocated for all copies of
33784e6437f1SZenghui Yu * the L1 table on *all* v4.1 RDs.
33794e6437f1SZenghui Yu */
33804e6437f1SZenghui Yu for_each_possible_cpu(cpu) {
33814e6437f1SZenghui Yu if (!allocate_vpe_l2_table(cpu, vpe_id))
33824e6437f1SZenghui Yu return false;
33834e6437f1SZenghui Yu }
33844e6437f1SZenghui Yu
33857d75bbb4SMarc Zyngier return true;
33867d75bbb4SMarc Zyngier }
33877d75bbb4SMarc Zyngier
its_create_device(struct its_node * its,u32 dev_id,int nvecs,bool alloc_lpis)338884a6a2e7SMarc Zyngier static struct its_device *its_create_device(struct its_node *its, u32 dev_id,
338993f94ea0SMarc Zyngier int nvecs, bool alloc_lpis)
339084a6a2e7SMarc Zyngier {
339184a6a2e7SMarc Zyngier struct its_device *dev;
339293f94ea0SMarc Zyngier unsigned long *lpi_map = NULL;
33933e39e8f5SMarc Zyngier unsigned long flags;
3394591e5becSMarc Zyngier u16 *col_map = NULL;
339584a6a2e7SMarc Zyngier void *itt;
339684a6a2e7SMarc Zyngier int lpi_base;
339784a6a2e7SMarc Zyngier int nr_lpis;
3398c8481267SMarc Zyngier int nr_ites;
339984a6a2e7SMarc Zyngier int sz;
340084a6a2e7SMarc Zyngier
34013faf24eaSShanker Donthineni if (!its_alloc_device_table(its, dev_id))
3402466b7d16SShanker Donthineni return NULL;
3403466b7d16SShanker Donthineni
3404147c8f37SMarc Zyngier if (WARN_ON(!is_power_of_2(nvecs)))
3405147c8f37SMarc Zyngier nvecs = roundup_pow_of_two(nvecs);
3406147c8f37SMarc Zyngier
340784a6a2e7SMarc Zyngier dev = kzalloc(sizeof(*dev), GFP_KERNEL);
3408c8481267SMarc Zyngier /*
3409147c8f37SMarc Zyngier * Even if the device wants a single LPI, the ITT must be
3410147c8f37SMarc Zyngier * sized as a power of two (and you need at least one bit...).
3411c8481267SMarc Zyngier */
3412147c8f37SMarc Zyngier nr_ites = max(2, nvecs);
3413ffedbf0cSMarc Zyngier sz = nr_ites * (FIELD_GET(GITS_TYPER_ITT_ENTRY_SIZE, its->typer) + 1);
341484a6a2e7SMarc Zyngier sz = max(sz, ITS_ITT_ALIGN) + ITS_ITT_ALIGN - 1;
3415539d3782SShanker Donthineni itt = kzalloc_node(sz, GFP_KERNEL, its->numa_node);
341693f94ea0SMarc Zyngier if (alloc_lpis) {
341738dd7c49SMarc Zyngier lpi_map = its_lpi_alloc(nvecs, &lpi_base, &nr_lpis);
3418591e5becSMarc Zyngier if (lpi_map)
34196396bb22SKees Cook col_map = kcalloc(nr_lpis, sizeof(*col_map),
342093f94ea0SMarc Zyngier GFP_KERNEL);
342193f94ea0SMarc Zyngier } else {
34226396bb22SKees Cook col_map = kcalloc(nr_ites, sizeof(*col_map), GFP_KERNEL);
342393f94ea0SMarc Zyngier nr_lpis = 0;
342493f94ea0SMarc Zyngier lpi_base = 0;
342593f94ea0SMarc Zyngier }
342684a6a2e7SMarc Zyngier
342793f94ea0SMarc Zyngier if (!dev || !itt || !col_map || (!lpi_map && alloc_lpis)) {
342884a6a2e7SMarc Zyngier kfree(dev);
342984a6a2e7SMarc Zyngier kfree(itt);
3430ff5fe886SAndy Shevchenko bitmap_free(lpi_map);
3431591e5becSMarc Zyngier kfree(col_map);
343284a6a2e7SMarc Zyngier return NULL;
343384a6a2e7SMarc Zyngier }
343484a6a2e7SMarc Zyngier
3435328191c0SVladimir Murzin gic_flush_dcache_to_poc(itt, sz);
34365a9a8915SMarc Zyngier
343784a6a2e7SMarc Zyngier dev->its = its;
343884a6a2e7SMarc Zyngier dev->itt = itt;
3439c8481267SMarc Zyngier dev->nr_ites = nr_ites;
3440591e5becSMarc Zyngier dev->event_map.lpi_map = lpi_map;
3441591e5becSMarc Zyngier dev->event_map.col_map = col_map;
3442591e5becSMarc Zyngier dev->event_map.lpi_base = lpi_base;
3443591e5becSMarc Zyngier dev->event_map.nr_lpis = nr_lpis;
344411635fa2SMarc Zyngier raw_spin_lock_init(&dev->event_map.vlpi_lock);
344584a6a2e7SMarc Zyngier dev->device_id = dev_id;
344684a6a2e7SMarc Zyngier INIT_LIST_HEAD(&dev->entry);
344784a6a2e7SMarc Zyngier
34483e39e8f5SMarc Zyngier raw_spin_lock_irqsave(&its->lock, flags);
344984a6a2e7SMarc Zyngier list_add(&dev->entry, &its->its_device_list);
34503e39e8f5SMarc Zyngier raw_spin_unlock_irqrestore(&its->lock, flags);
345184a6a2e7SMarc Zyngier
345284a6a2e7SMarc Zyngier /* Map device to its ITT */
345384a6a2e7SMarc Zyngier its_send_mapd(dev, 1);
345484a6a2e7SMarc Zyngier
345584a6a2e7SMarc Zyngier return dev;
345684a6a2e7SMarc Zyngier }
345784a6a2e7SMarc Zyngier
its_free_device(struct its_device * its_dev)345884a6a2e7SMarc Zyngier static void its_free_device(struct its_device *its_dev)
345984a6a2e7SMarc Zyngier {
34603e39e8f5SMarc Zyngier unsigned long flags;
34613e39e8f5SMarc Zyngier
34623e39e8f5SMarc Zyngier raw_spin_lock_irqsave(&its_dev->its->lock, flags);
346384a6a2e7SMarc Zyngier list_del(&its_dev->entry);
34643e39e8f5SMarc Zyngier raw_spin_unlock_irqrestore(&its_dev->its->lock, flags);
3465898aa5ceSMarc Zyngier kfree(its_dev->event_map.col_map);
346684a6a2e7SMarc Zyngier kfree(its_dev->itt);
346784a6a2e7SMarc Zyngier kfree(its_dev);
346884a6a2e7SMarc Zyngier }
3469b48ac83dSMarc Zyngier
its_alloc_device_irq(struct its_device * dev,int nvecs,irq_hw_number_t * hwirq)34708208d170SMarc Zyngier static int its_alloc_device_irq(struct its_device *dev, int nvecs, irq_hw_number_t *hwirq)
3471b48ac83dSMarc Zyngier {
3472b48ac83dSMarc Zyngier int idx;
3473b48ac83dSMarc Zyngier
3474342be106SZenghui Yu /* Find a free LPI region in lpi_map and allocate them. */
34758208d170SMarc Zyngier idx = bitmap_find_free_region(dev->event_map.lpi_map,
34768208d170SMarc Zyngier dev->event_map.nr_lpis,
34778208d170SMarc Zyngier get_count_order(nvecs));
34788208d170SMarc Zyngier if (idx < 0)
3479b48ac83dSMarc Zyngier return -ENOSPC;
3480b48ac83dSMarc Zyngier
3481591e5becSMarc Zyngier *hwirq = dev->event_map.lpi_base + idx;
3482b48ac83dSMarc Zyngier
3483b48ac83dSMarc Zyngier return 0;
3484b48ac83dSMarc Zyngier }
3485b48ac83dSMarc Zyngier
its_msi_prepare(struct irq_domain * domain,struct device * dev,int nvec,msi_alloc_info_t * info)348654456db9SMarc Zyngier static int its_msi_prepare(struct irq_domain *domain, struct device *dev,
3487b48ac83dSMarc Zyngier int nvec, msi_alloc_info_t *info)
3488b48ac83dSMarc Zyngier {
3489b48ac83dSMarc Zyngier struct its_node *its;
3490b48ac83dSMarc Zyngier struct its_device *its_dev;
349154456db9SMarc Zyngier struct msi_domain_info *msi_info;
349254456db9SMarc Zyngier u32 dev_id;
34939791ec7dSMarc Zyngier int err = 0;
3494b48ac83dSMarc Zyngier
349554456db9SMarc Zyngier /*
3496a7c90f51SJulien Grall * We ignore "dev" entirely, and rely on the dev_id that has
349754456db9SMarc Zyngier * been passed via the scratchpad. This limits this domain's
349854456db9SMarc Zyngier * usefulness to upper layers that definitely know that they
349954456db9SMarc Zyngier * are built on top of the ITS.
350054456db9SMarc Zyngier */
350154456db9SMarc Zyngier dev_id = info->scratchpad[0].ul;
350254456db9SMarc Zyngier
350354456db9SMarc Zyngier msi_info = msi_get_domain_info(domain);
350454456db9SMarc Zyngier its = msi_info->data;
350554456db9SMarc Zyngier
350620b3d54eSMarc Zyngier if (!gic_rdists->has_direct_lpi &&
350720b3d54eSMarc Zyngier vpe_proxy.dev &&
350820b3d54eSMarc Zyngier vpe_proxy.dev->its == its &&
350920b3d54eSMarc Zyngier dev_id == vpe_proxy.dev->device_id) {
351020b3d54eSMarc Zyngier /* Bad luck. Get yourself a better implementation */
351120b3d54eSMarc Zyngier WARN_ONCE(1, "DevId %x clashes with GICv4 VPE proxy device\n",
351220b3d54eSMarc Zyngier dev_id);
351320b3d54eSMarc Zyngier return -EINVAL;
351420b3d54eSMarc Zyngier }
351520b3d54eSMarc Zyngier
35169791ec7dSMarc Zyngier mutex_lock(&its->dev_alloc_lock);
3517f130420eSMarc Zyngier its_dev = its_find_device(its, dev_id);
3518e8137f4fSMarc Zyngier if (its_dev) {
3519e8137f4fSMarc Zyngier /*
3520e8137f4fSMarc Zyngier * We already have seen this ID, probably through
3521e8137f4fSMarc Zyngier * another alias (PCI bridge of some sort). No need to
3522e8137f4fSMarc Zyngier * create the device.
3523e8137f4fSMarc Zyngier */
35249791ec7dSMarc Zyngier its_dev->shared = true;
3525f130420eSMarc Zyngier pr_debug("Reusing ITT for devID %x\n", dev_id);
3526e8137f4fSMarc Zyngier goto out;
3527e8137f4fSMarc Zyngier }
3528b48ac83dSMarc Zyngier
352993f94ea0SMarc Zyngier its_dev = its_create_device(its, dev_id, nvec, true);
35309791ec7dSMarc Zyngier if (!its_dev) {
35319791ec7dSMarc Zyngier err = -ENOMEM;
35329791ec7dSMarc Zyngier goto out;
35339791ec7dSMarc Zyngier }
3534b48ac83dSMarc Zyngier
35355fe71d27SMarc Zyngier if (info->flags & MSI_ALLOC_FLAGS_PROXY_DEVICE)
35365fe71d27SMarc Zyngier its_dev->shared = true;
35375fe71d27SMarc Zyngier
3538f130420eSMarc Zyngier pr_debug("ITT %d entries, %d bits\n", nvec, ilog2(nvec));
3539e8137f4fSMarc Zyngier out:
35409791ec7dSMarc Zyngier mutex_unlock(&its->dev_alloc_lock);
3541b48ac83dSMarc Zyngier info->scratchpad[0].ptr = its_dev;
35429791ec7dSMarc Zyngier return err;
3543b48ac83dSMarc Zyngier }
3544b48ac83dSMarc Zyngier
354554456db9SMarc Zyngier static struct msi_domain_ops its_msi_domain_ops = {
354654456db9SMarc Zyngier .msi_prepare = its_msi_prepare,
354754456db9SMarc Zyngier };
354854456db9SMarc Zyngier
its_irq_gic_domain_alloc(struct irq_domain * domain,unsigned int virq,irq_hw_number_t hwirq)3549b48ac83dSMarc Zyngier static int its_irq_gic_domain_alloc(struct irq_domain *domain,
3550b48ac83dSMarc Zyngier unsigned int virq,
3551b48ac83dSMarc Zyngier irq_hw_number_t hwirq)
3552b48ac83dSMarc Zyngier {
3553f833f57fSMarc Zyngier struct irq_fwspec fwspec;
3554b48ac83dSMarc Zyngier
3555f833f57fSMarc Zyngier if (irq_domain_get_of_node(domain->parent)) {
3556f833f57fSMarc Zyngier fwspec.fwnode = domain->parent->fwnode;
3557f833f57fSMarc Zyngier fwspec.param_count = 3;
3558f833f57fSMarc Zyngier fwspec.param[0] = GIC_IRQ_TYPE_LPI;
3559f833f57fSMarc Zyngier fwspec.param[1] = hwirq;
3560f833f57fSMarc Zyngier fwspec.param[2] = IRQ_TYPE_EDGE_RISING;
35613f010cf1STomasz Nowicki } else if (is_fwnode_irqchip(domain->parent->fwnode)) {
35623f010cf1STomasz Nowicki fwspec.fwnode = domain->parent->fwnode;
35633f010cf1STomasz Nowicki fwspec.param_count = 2;
35643f010cf1STomasz Nowicki fwspec.param[0] = hwirq;
35653f010cf1STomasz Nowicki fwspec.param[1] = IRQ_TYPE_EDGE_RISING;
3566f833f57fSMarc Zyngier } else {
3567f833f57fSMarc Zyngier return -EINVAL;
3568f833f57fSMarc Zyngier }
3569b48ac83dSMarc Zyngier
3570f833f57fSMarc Zyngier return irq_domain_alloc_irqs_parent(domain, virq, 1, &fwspec);
3571b48ac83dSMarc Zyngier }
3572b48ac83dSMarc Zyngier
its_irq_domain_alloc(struct irq_domain * domain,unsigned int virq,unsigned int nr_irqs,void * args)3573b48ac83dSMarc Zyngier static int its_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
3574b48ac83dSMarc Zyngier unsigned int nr_irqs, void *args)
3575b48ac83dSMarc Zyngier {
3576b48ac83dSMarc Zyngier msi_alloc_info_t *info = args;
3577b48ac83dSMarc Zyngier struct its_device *its_dev = info->scratchpad[0].ptr;
357835ae7df2SJulien Grall struct its_node *its = its_dev->its;
3579f0c7bacaSThomas Gleixner struct irq_data *irqd;
3580b48ac83dSMarc Zyngier irq_hw_number_t hwirq;
3581b48ac83dSMarc Zyngier int err;
3582b48ac83dSMarc Zyngier int i;
3583b48ac83dSMarc Zyngier
35848208d170SMarc Zyngier err = its_alloc_device_irq(its_dev, nr_irqs, &hwirq);
3585b48ac83dSMarc Zyngier if (err)
3586b48ac83dSMarc Zyngier return err;
3587b48ac83dSMarc Zyngier
358835ae7df2SJulien Grall err = iommu_dma_prepare_msi(info->desc, its->get_msi_base(its_dev));
358935ae7df2SJulien Grall if (err)
359035ae7df2SJulien Grall return err;
359135ae7df2SJulien Grall
35928208d170SMarc Zyngier for (i = 0; i < nr_irqs; i++) {
35938208d170SMarc Zyngier err = its_irq_gic_domain_alloc(domain, virq + i, hwirq + i);
3594b48ac83dSMarc Zyngier if (err)
3595b48ac83dSMarc Zyngier return err;
3596b48ac83dSMarc Zyngier
3597b48ac83dSMarc Zyngier irq_domain_set_hwirq_and_chip(domain, virq + i,
35988208d170SMarc Zyngier hwirq + i, &its_irq_chip, its_dev);
3599f0c7bacaSThomas Gleixner irqd = irq_get_irq_data(virq + i);
3600f0c7bacaSThomas Gleixner irqd_set_single_target(irqd);
3601f0c7bacaSThomas Gleixner irqd_set_affinity_on_activate(irqd);
36028f4b5895SJames Gowans irqd_set_resend_when_in_progress(irqd);
3603f130420eSMarc Zyngier pr_debug("ID:%d pID:%d vID:%d\n",
36048208d170SMarc Zyngier (int)(hwirq + i - its_dev->event_map.lpi_base),
36058208d170SMarc Zyngier (int)(hwirq + i), virq + i);
3606b48ac83dSMarc Zyngier }
3607b48ac83dSMarc Zyngier
3608b48ac83dSMarc Zyngier return 0;
3609b48ac83dSMarc Zyngier }
3610b48ac83dSMarc Zyngier
its_irq_domain_activate(struct irq_domain * domain,struct irq_data * d,bool reserve)361172491643SThomas Gleixner static int its_irq_domain_activate(struct irq_domain *domain,
3612702cb0a0SThomas Gleixner struct irq_data *d, bool reserve)
3613aca268dfSMarc Zyngier {
3614aca268dfSMarc Zyngier struct its_device *its_dev = irq_data_get_irq_chip_data(d);
3615aca268dfSMarc Zyngier u32 event = its_get_event_id(d);
36160d224d35SMarc Zyngier int cpu;
3617fbf8f40eSGanapatrao Kulkarni
3618c5d6082dSMarc Zyngier cpu = its_select_cpu(d, cpu_online_mask);
3619c5d6082dSMarc Zyngier if (cpu < 0 || cpu >= nr_cpu_ids)
3620c1797b11SYang Yingliang return -EINVAL;
3621c1797b11SYang Yingliang
36222f13ff1dSMarc Zyngier its_inc_lpi_count(d, cpu);
36230d224d35SMarc Zyngier its_dev->event_map.col_map[event] = cpu;
36240d224d35SMarc Zyngier irq_data_update_effective_affinity(d, cpumask_of(cpu));
3625591e5becSMarc Zyngier
3626aca268dfSMarc Zyngier /* Map the GIC IRQ and event to the device */
36276a25ad3aSMarc Zyngier its_send_mapti(its_dev, d->hwirq, event);
362872491643SThomas Gleixner return 0;
3629aca268dfSMarc Zyngier }
3630aca268dfSMarc Zyngier
its_irq_domain_deactivate(struct irq_domain * domain,struct irq_data * d)3631aca268dfSMarc Zyngier static void its_irq_domain_deactivate(struct irq_domain *domain,
3632aca268dfSMarc Zyngier struct irq_data *d)
3633aca268dfSMarc Zyngier {
3634aca268dfSMarc Zyngier struct its_device *its_dev = irq_data_get_irq_chip_data(d);
3635aca268dfSMarc Zyngier u32 event = its_get_event_id(d);
3636aca268dfSMarc Zyngier
36372f13ff1dSMarc Zyngier its_dec_lpi_count(d, its_dev->event_map.col_map[event]);
3638aca268dfSMarc Zyngier /* Stop the delivery of interrupts */
3639aca268dfSMarc Zyngier its_send_discard(its_dev, event);
3640aca268dfSMarc Zyngier }
3641aca268dfSMarc Zyngier
its_irq_domain_free(struct irq_domain * domain,unsigned int virq,unsigned int nr_irqs)3642b48ac83dSMarc Zyngier static void its_irq_domain_free(struct irq_domain *domain, unsigned int virq,
3643b48ac83dSMarc Zyngier unsigned int nr_irqs)
3644b48ac83dSMarc Zyngier {
3645b48ac83dSMarc Zyngier struct irq_data *d = irq_domain_get_irq_data(domain, virq);
3646b48ac83dSMarc Zyngier struct its_device *its_dev = irq_data_get_irq_chip_data(d);
36479791ec7dSMarc Zyngier struct its_node *its = its_dev->its;
3648b48ac83dSMarc Zyngier int i;
3649b48ac83dSMarc Zyngier
3650c9c96e30SMarc Zyngier bitmap_release_region(its_dev->event_map.lpi_map,
3651c9c96e30SMarc Zyngier its_get_event_id(irq_domain_get_irq_data(domain, virq)),
3652c9c96e30SMarc Zyngier get_count_order(nr_irqs));
3653c9c96e30SMarc Zyngier
3654b48ac83dSMarc Zyngier for (i = 0; i < nr_irqs; i++) {
3655b48ac83dSMarc Zyngier struct irq_data *data = irq_domain_get_irq_data(domain,
3656b48ac83dSMarc Zyngier virq + i);
3657b48ac83dSMarc Zyngier /* Nuke the entry in the domain */
36582da39949SMarc Zyngier irq_domain_reset_irq_data(data);
3659b48ac83dSMarc Zyngier }
3660b48ac83dSMarc Zyngier
36619791ec7dSMarc Zyngier mutex_lock(&its->dev_alloc_lock);
36629791ec7dSMarc Zyngier
36639791ec7dSMarc Zyngier /*
36649791ec7dSMarc Zyngier * If all interrupts have been freed, start mopping the
3665a359f757SIngo Molnar * floor. This is conditioned on the device not being shared.
36669791ec7dSMarc Zyngier */
36679791ec7dSMarc Zyngier if (!its_dev->shared &&
36689791ec7dSMarc Zyngier bitmap_empty(its_dev->event_map.lpi_map,
3669591e5becSMarc Zyngier its_dev->event_map.nr_lpis)) {
367038dd7c49SMarc Zyngier its_lpi_free(its_dev->event_map.lpi_map,
3671cf2be8baSMarc Zyngier its_dev->event_map.lpi_base,
3672cf2be8baSMarc Zyngier its_dev->event_map.nr_lpis);
3673b48ac83dSMarc Zyngier
3674b48ac83dSMarc Zyngier /* Unmap device/itt */
3675b48ac83dSMarc Zyngier its_send_mapd(its_dev, 0);
3676b48ac83dSMarc Zyngier its_free_device(its_dev);
3677b48ac83dSMarc Zyngier }
3678b48ac83dSMarc Zyngier
36799791ec7dSMarc Zyngier mutex_unlock(&its->dev_alloc_lock);
36809791ec7dSMarc Zyngier
3681b48ac83dSMarc Zyngier irq_domain_free_irqs_parent(domain, virq, nr_irqs);
3682b48ac83dSMarc Zyngier }
3683b48ac83dSMarc Zyngier
3684b48ac83dSMarc Zyngier static const struct irq_domain_ops its_domain_ops = {
3685b48ac83dSMarc Zyngier .alloc = its_irq_domain_alloc,
3686b48ac83dSMarc Zyngier .free = its_irq_domain_free,
3687aca268dfSMarc Zyngier .activate = its_irq_domain_activate,
3688aca268dfSMarc Zyngier .deactivate = its_irq_domain_deactivate,
3689b48ac83dSMarc Zyngier };
36904c21f3c2SMarc Zyngier
369120b3d54eSMarc Zyngier /*
369220b3d54eSMarc Zyngier * This is insane.
369320b3d54eSMarc Zyngier *
36940684c704SMarc Zyngier * If a GICv4.0 doesn't implement Direct LPIs (which is extremely
369520b3d54eSMarc Zyngier * likely), the only way to perform an invalidate is to use a fake
369620b3d54eSMarc Zyngier * device to issue an INV command, implying that the LPI has first
369720b3d54eSMarc Zyngier * been mapped to some event on that device. Since this is not exactly
369820b3d54eSMarc Zyngier * cheap, we try to keep that mapping around as long as possible, and
369920b3d54eSMarc Zyngier * only issue an UNMAP if we're short on available slots.
370020b3d54eSMarc Zyngier *
370120b3d54eSMarc Zyngier * Broken by design(tm).
37020684c704SMarc Zyngier *
37030684c704SMarc Zyngier * GICv4.1, on the other hand, mandates that we're able to invalidate
37040684c704SMarc Zyngier * by writing to a MMIO register. It doesn't implement the whole of
37050684c704SMarc Zyngier * DirectLPI, but that's good enough. And most of the time, we don't
37060684c704SMarc Zyngier * even have to invalidate anything, as the redistributor can be told
37070684c704SMarc Zyngier * whether to generate a doorbell or not (we thus leave it enabled,
37080684c704SMarc Zyngier * always).
370920b3d54eSMarc Zyngier */
its_vpe_db_proxy_unmap_locked(struct its_vpe * vpe)371020b3d54eSMarc Zyngier static void its_vpe_db_proxy_unmap_locked(struct its_vpe *vpe)
371120b3d54eSMarc Zyngier {
37120684c704SMarc Zyngier /* GICv4.1 doesn't use a proxy, so nothing to do here */
37130684c704SMarc Zyngier if (gic_rdists->has_rvpeid)
37140684c704SMarc Zyngier return;
37150684c704SMarc Zyngier
371620b3d54eSMarc Zyngier /* Already unmapped? */
371720b3d54eSMarc Zyngier if (vpe->vpe_proxy_event == -1)
371820b3d54eSMarc Zyngier return;
371920b3d54eSMarc Zyngier
372020b3d54eSMarc Zyngier its_send_discard(vpe_proxy.dev, vpe->vpe_proxy_event);
372120b3d54eSMarc Zyngier vpe_proxy.vpes[vpe->vpe_proxy_event] = NULL;
372220b3d54eSMarc Zyngier
372320b3d54eSMarc Zyngier /*
372420b3d54eSMarc Zyngier * We don't track empty slots at all, so let's move the
372520b3d54eSMarc Zyngier * next_victim pointer if we can quickly reuse that slot
372620b3d54eSMarc Zyngier * instead of nuking an existing entry. Not clear that this is
372720b3d54eSMarc Zyngier * always a win though, and this might just generate a ripple
372820b3d54eSMarc Zyngier * effect... Let's just hope VPEs don't migrate too often.
372920b3d54eSMarc Zyngier */
373020b3d54eSMarc Zyngier if (vpe_proxy.vpes[vpe_proxy.next_victim])
373120b3d54eSMarc Zyngier vpe_proxy.next_victim = vpe->vpe_proxy_event;
373220b3d54eSMarc Zyngier
373320b3d54eSMarc Zyngier vpe->vpe_proxy_event = -1;
373420b3d54eSMarc Zyngier }
373520b3d54eSMarc Zyngier
its_vpe_db_proxy_unmap(struct its_vpe * vpe)373620b3d54eSMarc Zyngier static void its_vpe_db_proxy_unmap(struct its_vpe *vpe)
373720b3d54eSMarc Zyngier {
37380684c704SMarc Zyngier /* GICv4.1 doesn't use a proxy, so nothing to do here */
37390684c704SMarc Zyngier if (gic_rdists->has_rvpeid)
37400684c704SMarc Zyngier return;
37410684c704SMarc Zyngier
374220b3d54eSMarc Zyngier if (!gic_rdists->has_direct_lpi) {
374320b3d54eSMarc Zyngier unsigned long flags;
374420b3d54eSMarc Zyngier
374520b3d54eSMarc Zyngier raw_spin_lock_irqsave(&vpe_proxy.lock, flags);
374620b3d54eSMarc Zyngier its_vpe_db_proxy_unmap_locked(vpe);
374720b3d54eSMarc Zyngier raw_spin_unlock_irqrestore(&vpe_proxy.lock, flags);
374820b3d54eSMarc Zyngier }
374920b3d54eSMarc Zyngier }
375020b3d54eSMarc Zyngier
its_vpe_db_proxy_map_locked(struct its_vpe * vpe)375120b3d54eSMarc Zyngier static void its_vpe_db_proxy_map_locked(struct its_vpe *vpe)
375220b3d54eSMarc Zyngier {
37530684c704SMarc Zyngier /* GICv4.1 doesn't use a proxy, so nothing to do here */
37540684c704SMarc Zyngier if (gic_rdists->has_rvpeid)
37550684c704SMarc Zyngier return;
37560684c704SMarc Zyngier
375720b3d54eSMarc Zyngier /* Already mapped? */
375820b3d54eSMarc Zyngier if (vpe->vpe_proxy_event != -1)
375920b3d54eSMarc Zyngier return;
376020b3d54eSMarc Zyngier
376120b3d54eSMarc Zyngier /* This slot was already allocated. Kick the other VPE out. */
376220b3d54eSMarc Zyngier if (vpe_proxy.vpes[vpe_proxy.next_victim])
376320b3d54eSMarc Zyngier its_vpe_db_proxy_unmap_locked(vpe_proxy.vpes[vpe_proxy.next_victim]);
376420b3d54eSMarc Zyngier
376520b3d54eSMarc Zyngier /* Map the new VPE instead */
376620b3d54eSMarc Zyngier vpe_proxy.vpes[vpe_proxy.next_victim] = vpe;
376720b3d54eSMarc Zyngier vpe->vpe_proxy_event = vpe_proxy.next_victim;
376820b3d54eSMarc Zyngier vpe_proxy.next_victim = (vpe_proxy.next_victim + 1) % vpe_proxy.dev->nr_ites;
376920b3d54eSMarc Zyngier
377020b3d54eSMarc Zyngier vpe_proxy.dev->event_map.col_map[vpe->vpe_proxy_event] = vpe->col_idx;
377120b3d54eSMarc Zyngier its_send_mapti(vpe_proxy.dev, vpe->vpe_db_lpi, vpe->vpe_proxy_event);
377220b3d54eSMarc Zyngier }
377320b3d54eSMarc Zyngier
its_vpe_db_proxy_move(struct its_vpe * vpe,int from,int to)3774958b90d1SMarc Zyngier static void its_vpe_db_proxy_move(struct its_vpe *vpe, int from, int to)
3775958b90d1SMarc Zyngier {
3776958b90d1SMarc Zyngier unsigned long flags;
3777958b90d1SMarc Zyngier struct its_collection *target_col;
3778958b90d1SMarc Zyngier
37790684c704SMarc Zyngier /* GICv4.1 doesn't use a proxy, so nothing to do here */
37800684c704SMarc Zyngier if (gic_rdists->has_rvpeid)
37810684c704SMarc Zyngier return;
37820684c704SMarc Zyngier
3783958b90d1SMarc Zyngier if (gic_rdists->has_direct_lpi) {
3784958b90d1SMarc Zyngier void __iomem *rdbase;
3785958b90d1SMarc Zyngier
3786958b90d1SMarc Zyngier rdbase = per_cpu_ptr(gic_rdists->rdist, from)->rd_base;
3787958b90d1SMarc Zyngier gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_CLRLPIR);
37882f4f064bSMarc Zyngier wait_for_syncr(rdbase);
3789958b90d1SMarc Zyngier
3790958b90d1SMarc Zyngier return;
3791958b90d1SMarc Zyngier }
3792958b90d1SMarc Zyngier
3793958b90d1SMarc Zyngier raw_spin_lock_irqsave(&vpe_proxy.lock, flags);
3794958b90d1SMarc Zyngier
3795958b90d1SMarc Zyngier its_vpe_db_proxy_map_locked(vpe);
3796958b90d1SMarc Zyngier
3797958b90d1SMarc Zyngier target_col = &vpe_proxy.dev->its->collections[to];
3798958b90d1SMarc Zyngier its_send_movi(vpe_proxy.dev, target_col, vpe->vpe_proxy_event);
3799958b90d1SMarc Zyngier vpe_proxy.dev->event_map.col_map[vpe->vpe_proxy_event] = to;
3800958b90d1SMarc Zyngier
3801958b90d1SMarc Zyngier raw_spin_unlock_irqrestore(&vpe_proxy.lock, flags);
3802958b90d1SMarc Zyngier }
3803958b90d1SMarc Zyngier
its_vpe_set_affinity(struct irq_data * d,const struct cpumask * mask_val,bool force)38043171a47aSMarc Zyngier static int its_vpe_set_affinity(struct irq_data *d,
38053171a47aSMarc Zyngier const struct cpumask *mask_val,
38063171a47aSMarc Zyngier bool force)
38073171a47aSMarc Zyngier {
38083171a47aSMarc Zyngier struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
380965ac3a4fSMarc Zyngier struct cpumask common, *table_mask;
3810f3a05921SMarc Zyngier unsigned long flags;
381165ac3a4fSMarc Zyngier int from, cpu;
38123171a47aSMarc Zyngier
38133171a47aSMarc Zyngier /*
38143171a47aSMarc Zyngier * Changing affinity is mega expensive, so let's be as lazy as
381520b3d54eSMarc Zyngier * we can and only do it if we really have to. Also, if mapped
3816958b90d1SMarc Zyngier * into the proxy device, we need to move the doorbell
3817958b90d1SMarc Zyngier * interrupt to its new location.
3818f3a05921SMarc Zyngier *
3819f3a05921SMarc Zyngier * Another thing is that changing the affinity of a vPE affects
3820f3a05921SMarc Zyngier * *other interrupts* such as all the vLPIs that are routed to
3821f3a05921SMarc Zyngier * this vPE. This means that the irq_desc lock is not enough to
3822f3a05921SMarc Zyngier * protect us, and that we must ensure nobody samples vpe->col_idx
3823f3a05921SMarc Zyngier * during the update, hence the lock below which must also be
3824f3a05921SMarc Zyngier * taken on any vLPI handling path that evaluates vpe->col_idx.
38253171a47aSMarc Zyngier */
3826f3a05921SMarc Zyngier from = vpe_to_cpuid_lock(vpe, &flags);
382765ac3a4fSMarc Zyngier table_mask = gic_data_rdist_cpu(from)->vpe_table_mask;
382865ac3a4fSMarc Zyngier
382965ac3a4fSMarc Zyngier /*
383065ac3a4fSMarc Zyngier * If we are offered another CPU in the same GICv4.1 ITS
383165ac3a4fSMarc Zyngier * affinity, pick this one. Otherwise, any CPU will do.
383265ac3a4fSMarc Zyngier */
383365ac3a4fSMarc Zyngier if (table_mask && cpumask_and(&common, mask_val, table_mask))
383465ac3a4fSMarc Zyngier cpu = cpumask_test_cpu(from, &common) ? from : cpumask_first(&common);
383565ac3a4fSMarc Zyngier else
383665ac3a4fSMarc Zyngier cpu = cpumask_first(mask_val);
383765ac3a4fSMarc Zyngier
3838f3a05921SMarc Zyngier if (from == cpu)
3839dd3f050aSMarc Zyngier goto out;
3840958b90d1SMarc Zyngier
38413171a47aSMarc Zyngier vpe->col_idx = cpu;
3842dd3f050aSMarc Zyngier
38433171a47aSMarc Zyngier its_send_vmovp(vpe);
3844958b90d1SMarc Zyngier its_vpe_db_proxy_move(vpe, from, cpu);
38453171a47aSMarc Zyngier
3846dd3f050aSMarc Zyngier out:
384744c4c25eSMarc Zyngier irq_data_update_effective_affinity(d, cpumask_of(cpu));
3848f3a05921SMarc Zyngier vpe_to_cpuid_unlock(vpe, flags);
384944c4c25eSMarc Zyngier
38503171a47aSMarc Zyngier return IRQ_SET_MASK_OK_DONE;
38513171a47aSMarc Zyngier }
38523171a47aSMarc Zyngier
its_wait_vpt_parse_complete(void)385396806229SMarc Zyngier static void its_wait_vpt_parse_complete(void)
385496806229SMarc Zyngier {
385596806229SMarc Zyngier void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
385696806229SMarc Zyngier u64 val;
385796806229SMarc Zyngier
385896806229SMarc Zyngier if (!gic_rdists->has_vpend_valid_dirty)
385996806229SMarc Zyngier return;
386096806229SMarc Zyngier
386131dbb6b1SZenghui Yu WARN_ON_ONCE(readq_relaxed_poll_timeout_atomic(vlpi_base + GICR_VPENDBASER,
386296806229SMarc Zyngier val,
386396806229SMarc Zyngier !(val & GICR_VPENDBASER_Dirty),
38640b394982SShenming Lu 1, 500));
386596806229SMarc Zyngier }
386696806229SMarc Zyngier
its_vpe_schedule(struct its_vpe * vpe)3867e643d803SMarc Zyngier static void its_vpe_schedule(struct its_vpe *vpe)
3868e643d803SMarc Zyngier {
386950c33097SRobin Murphy void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
3870e643d803SMarc Zyngier u64 val;
3871e643d803SMarc Zyngier
3872e643d803SMarc Zyngier /* Schedule the VPE */
3873e643d803SMarc Zyngier val = virt_to_phys(page_address(vpe->its_vm->vprop_page)) &
3874e643d803SMarc Zyngier GENMASK_ULL(51, 12);
3875e643d803SMarc Zyngier val |= (LPI_NRBITS - 1) & GICR_VPROPBASER_IDBITS_MASK;
3876f7e84c8eSMarc Zyngier if (rdists_support_shareable()) {
3877e643d803SMarc Zyngier val |= GICR_VPROPBASER_RaWb;
3878e643d803SMarc Zyngier val |= GICR_VPROPBASER_InnerShareable;
3879f7e84c8eSMarc Zyngier }
38805186a6ccSZenghui Yu gicr_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER);
3881e643d803SMarc Zyngier
3882e643d803SMarc Zyngier val = virt_to_phys(page_address(vpe->vpt_page)) &
3883e643d803SMarc Zyngier GENMASK_ULL(51, 16);
3884f7e84c8eSMarc Zyngier if (rdists_support_shareable()) {
3885e643d803SMarc Zyngier val |= GICR_VPENDBASER_RaWaWb;
3886b2cb11f4SHeyi Guo val |= GICR_VPENDBASER_InnerShareable;
3887f7e84c8eSMarc Zyngier }
3888e643d803SMarc Zyngier /*
3889e643d803SMarc Zyngier * There is no good way of finding out if the pending table is
3890e643d803SMarc Zyngier * empty as we can race against the doorbell interrupt very
3891e643d803SMarc Zyngier * easily. So in the end, vpe->pending_last is only an
3892e643d803SMarc Zyngier * indication that the vcpu has something pending, not one
3893e643d803SMarc Zyngier * that the pending table is empty. A good implementation
3894e643d803SMarc Zyngier * would be able to read its coarse map pretty quickly anyway,
3895e643d803SMarc Zyngier * making this a tolerable issue.
3896e643d803SMarc Zyngier */
3897e643d803SMarc Zyngier val |= GICR_VPENDBASER_PendingLast;
3898e643d803SMarc Zyngier val |= vpe->idai ? GICR_VPENDBASER_IDAI : 0;
3899e643d803SMarc Zyngier val |= GICR_VPENDBASER_Valid;
39005186a6ccSZenghui Yu gicr_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER);
3901e643d803SMarc Zyngier }
3902e643d803SMarc Zyngier
its_vpe_deschedule(struct its_vpe * vpe)3903e643d803SMarc Zyngier static void its_vpe_deschedule(struct its_vpe *vpe)
3904e643d803SMarc Zyngier {
390550c33097SRobin Murphy void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
3906e643d803SMarc Zyngier u64 val;
3907e643d803SMarc Zyngier
3908e64fab1aSMarc Zyngier val = its_clear_vpend_valid(vlpi_base, 0, 0);
3909e643d803SMarc Zyngier
3910e643d803SMarc Zyngier vpe->idai = !!(val & GICR_VPENDBASER_IDAI);
3911e643d803SMarc Zyngier vpe->pending_last = !!(val & GICR_VPENDBASER_PendingLast);
3912e643d803SMarc Zyngier }
3913e643d803SMarc Zyngier
its_vpe_invall(struct its_vpe * vpe)391440619a2eSMarc Zyngier static void its_vpe_invall(struct its_vpe *vpe)
391540619a2eSMarc Zyngier {
391640619a2eSMarc Zyngier struct its_node *its;
391740619a2eSMarc Zyngier
391840619a2eSMarc Zyngier list_for_each_entry(its, &its_nodes, entry) {
39190dd57fedSMarc Zyngier if (!is_v4(its))
392040619a2eSMarc Zyngier continue;
392140619a2eSMarc Zyngier
39222247e1bfSMarc Zyngier if (its_list_map && !vpe->its_vm->vlpi_count[its->list_nr])
39232247e1bfSMarc Zyngier continue;
39242247e1bfSMarc Zyngier
39253c1cceebSMarc Zyngier /*
39263c1cceebSMarc Zyngier * Sending a VINVALL to a single ITS is enough, as all
39273c1cceebSMarc Zyngier * we need is to reach the redistributors.
39283c1cceebSMarc Zyngier */
392940619a2eSMarc Zyngier its_send_vinvall(its, vpe);
39303c1cceebSMarc Zyngier return;
393140619a2eSMarc Zyngier }
393240619a2eSMarc Zyngier }
393340619a2eSMarc Zyngier
its_vpe_set_vcpu_affinity(struct irq_data * d,void * vcpu_info)3934e643d803SMarc Zyngier static int its_vpe_set_vcpu_affinity(struct irq_data *d, void *vcpu_info)
3935e643d803SMarc Zyngier {
3936e643d803SMarc Zyngier struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
3937e643d803SMarc Zyngier struct its_cmd_info *info = vcpu_info;
3938e643d803SMarc Zyngier
3939e643d803SMarc Zyngier switch (info->cmd_type) {
3940e643d803SMarc Zyngier case SCHEDULE_VPE:
3941e643d803SMarc Zyngier its_vpe_schedule(vpe);
3942e643d803SMarc Zyngier return 0;
3943e643d803SMarc Zyngier
3944e643d803SMarc Zyngier case DESCHEDULE_VPE:
3945e643d803SMarc Zyngier its_vpe_deschedule(vpe);
3946e643d803SMarc Zyngier return 0;
3947e643d803SMarc Zyngier
394857e3cebdSShenming Lu case COMMIT_VPE:
394957e3cebdSShenming Lu its_wait_vpt_parse_complete();
395057e3cebdSShenming Lu return 0;
395157e3cebdSShenming Lu
39525e2f7642SMarc Zyngier case INVALL_VPE:
395340619a2eSMarc Zyngier its_vpe_invall(vpe);
39545e2f7642SMarc Zyngier return 0;
39555e2f7642SMarc Zyngier
3956e643d803SMarc Zyngier default:
3957e643d803SMarc Zyngier return -EINVAL;
3958e643d803SMarc Zyngier }
3959e643d803SMarc Zyngier }
3960e643d803SMarc Zyngier
its_vpe_send_cmd(struct its_vpe * vpe,void (* cmd)(struct its_device *,u32))396120b3d54eSMarc Zyngier static void its_vpe_send_cmd(struct its_vpe *vpe,
396220b3d54eSMarc Zyngier void (*cmd)(struct its_device *, u32))
396320b3d54eSMarc Zyngier {
396420b3d54eSMarc Zyngier unsigned long flags;
396520b3d54eSMarc Zyngier
396620b3d54eSMarc Zyngier raw_spin_lock_irqsave(&vpe_proxy.lock, flags);
396720b3d54eSMarc Zyngier
396820b3d54eSMarc Zyngier its_vpe_db_proxy_map_locked(vpe);
396920b3d54eSMarc Zyngier cmd(vpe_proxy.dev, vpe->vpe_proxy_event);
397020b3d54eSMarc Zyngier
397120b3d54eSMarc Zyngier raw_spin_unlock_irqrestore(&vpe_proxy.lock, flags);
397220b3d54eSMarc Zyngier }
397320b3d54eSMarc Zyngier
its_vpe_send_inv(struct irq_data * d)3974f6a91da7SMarc Zyngier static void its_vpe_send_inv(struct irq_data *d)
3975f6a91da7SMarc Zyngier {
3976f6a91da7SMarc Zyngier struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
397720b3d54eSMarc Zyngier
3978926846a7SMarc Zyngier if (gic_rdists->has_direct_lpi)
3979926846a7SMarc Zyngier __direct_lpi_inv(d, d->parent_data->hwirq);
3980926846a7SMarc Zyngier else
398120b3d54eSMarc Zyngier its_vpe_send_cmd(vpe, its_send_inv);
398220b3d54eSMarc Zyngier }
3983f6a91da7SMarc Zyngier
its_vpe_mask_irq(struct irq_data * d)3984f6a91da7SMarc Zyngier static void its_vpe_mask_irq(struct irq_data *d)
3985f6a91da7SMarc Zyngier {
3986f6a91da7SMarc Zyngier /*
3987f6a91da7SMarc Zyngier * We need to unmask the LPI, which is described by the parent
3988f6a91da7SMarc Zyngier * irq_data. Instead of calling into the parent (which won't
3989f6a91da7SMarc Zyngier * exactly do the right thing, let's simply use the
3990f6a91da7SMarc Zyngier * parent_data pointer. Yes, I'm naughty.
3991f6a91da7SMarc Zyngier */
3992f6a91da7SMarc Zyngier lpi_write_config(d->parent_data, LPI_PROP_ENABLED, 0);
3993f6a91da7SMarc Zyngier its_vpe_send_inv(d);
3994f6a91da7SMarc Zyngier }
3995f6a91da7SMarc Zyngier
its_vpe_unmask_irq(struct irq_data * d)3996f6a91da7SMarc Zyngier static void its_vpe_unmask_irq(struct irq_data *d)
3997f6a91da7SMarc Zyngier {
3998f6a91da7SMarc Zyngier /* Same hack as above... */
3999f6a91da7SMarc Zyngier lpi_write_config(d->parent_data, 0, LPI_PROP_ENABLED);
4000f6a91da7SMarc Zyngier its_vpe_send_inv(d);
4001f6a91da7SMarc Zyngier }
4002f6a91da7SMarc Zyngier
its_vpe_set_irqchip_state(struct irq_data * d,enum irqchip_irq_state which,bool state)4003e57a3e28SMarc Zyngier static int its_vpe_set_irqchip_state(struct irq_data *d,
4004e57a3e28SMarc Zyngier enum irqchip_irq_state which,
4005e57a3e28SMarc Zyngier bool state)
4006e57a3e28SMarc Zyngier {
4007e57a3e28SMarc Zyngier struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4008e57a3e28SMarc Zyngier
4009e57a3e28SMarc Zyngier if (which != IRQCHIP_STATE_PENDING)
4010e57a3e28SMarc Zyngier return -EINVAL;
4011e57a3e28SMarc Zyngier
4012e57a3e28SMarc Zyngier if (gic_rdists->has_direct_lpi) {
4013e57a3e28SMarc Zyngier void __iomem *rdbase;
4014e57a3e28SMarc Zyngier
4015e57a3e28SMarc Zyngier rdbase = per_cpu_ptr(gic_rdists->rdist, vpe->col_idx)->rd_base;
4016e57a3e28SMarc Zyngier if (state) {
4017e57a3e28SMarc Zyngier gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_SETLPIR);
4018e57a3e28SMarc Zyngier } else {
4019e57a3e28SMarc Zyngier gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_CLRLPIR);
40202f4f064bSMarc Zyngier wait_for_syncr(rdbase);
4021e57a3e28SMarc Zyngier }
4022e57a3e28SMarc Zyngier } else {
4023e57a3e28SMarc Zyngier if (state)
4024e57a3e28SMarc Zyngier its_vpe_send_cmd(vpe, its_send_int);
4025e57a3e28SMarc Zyngier else
4026e57a3e28SMarc Zyngier its_vpe_send_cmd(vpe, its_send_clear);
4027e57a3e28SMarc Zyngier }
4028e57a3e28SMarc Zyngier
4029e57a3e28SMarc Zyngier return 0;
4030e57a3e28SMarc Zyngier }
4031e57a3e28SMarc Zyngier
its_vpe_retrigger(struct irq_data * d)40327809f701SMarc Zyngier static int its_vpe_retrigger(struct irq_data *d)
40337809f701SMarc Zyngier {
40347809f701SMarc Zyngier return !its_vpe_set_irqchip_state(d, IRQCHIP_STATE_PENDING, true);
40357809f701SMarc Zyngier }
40367809f701SMarc Zyngier
40378fff27aeSMarc Zyngier static struct irq_chip its_vpe_irq_chip = {
40388fff27aeSMarc Zyngier .name = "GICv4-vpe",
4039f6a91da7SMarc Zyngier .irq_mask = its_vpe_mask_irq,
4040f6a91da7SMarc Zyngier .irq_unmask = its_vpe_unmask_irq,
4041f6a91da7SMarc Zyngier .irq_eoi = irq_chip_eoi_parent,
40423171a47aSMarc Zyngier .irq_set_affinity = its_vpe_set_affinity,
40437809f701SMarc Zyngier .irq_retrigger = its_vpe_retrigger,
4044e57a3e28SMarc Zyngier .irq_set_irqchip_state = its_vpe_set_irqchip_state,
4045e643d803SMarc Zyngier .irq_set_vcpu_affinity = its_vpe_set_vcpu_affinity,
40468fff27aeSMarc Zyngier };
40478fff27aeSMarc Zyngier
find_4_1_its(void)4048d97c97baSMarc Zyngier static struct its_node *find_4_1_its(void)
4049d97c97baSMarc Zyngier {
4050d97c97baSMarc Zyngier static struct its_node *its = NULL;
4051d97c97baSMarc Zyngier
4052d97c97baSMarc Zyngier if (!its) {
4053d97c97baSMarc Zyngier list_for_each_entry(its, &its_nodes, entry) {
4054d97c97baSMarc Zyngier if (is_v4_1(its))
4055d97c97baSMarc Zyngier return its;
4056d97c97baSMarc Zyngier }
4057d97c97baSMarc Zyngier
4058d97c97baSMarc Zyngier /* Oops? */
4059d97c97baSMarc Zyngier its = NULL;
4060d97c97baSMarc Zyngier }
4061d97c97baSMarc Zyngier
4062d97c97baSMarc Zyngier return its;
4063d97c97baSMarc Zyngier }
4064d97c97baSMarc Zyngier
its_vpe_4_1_send_inv(struct irq_data * d)4065d97c97baSMarc Zyngier static void its_vpe_4_1_send_inv(struct irq_data *d)
4066d97c97baSMarc Zyngier {
4067d97c97baSMarc Zyngier struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4068d97c97baSMarc Zyngier struct its_node *its;
4069d97c97baSMarc Zyngier
4070d97c97baSMarc Zyngier /*
4071d97c97baSMarc Zyngier * GICv4.1 wants doorbells to be invalidated using the
4072d97c97baSMarc Zyngier * INVDB command in order to be broadcast to all RDs. Send
4073d97c97baSMarc Zyngier * it to the first valid ITS, and let the HW do its magic.
4074d97c97baSMarc Zyngier */
4075d97c97baSMarc Zyngier its = find_4_1_its();
4076d97c97baSMarc Zyngier if (its)
4077d97c97baSMarc Zyngier its_send_invdb(its, vpe);
4078d97c97baSMarc Zyngier }
4079d97c97baSMarc Zyngier
its_vpe_4_1_mask_irq(struct irq_data * d)4080d97c97baSMarc Zyngier static void its_vpe_4_1_mask_irq(struct irq_data *d)
4081d97c97baSMarc Zyngier {
4082d97c97baSMarc Zyngier lpi_write_config(d->parent_data, LPI_PROP_ENABLED, 0);
4083d97c97baSMarc Zyngier its_vpe_4_1_send_inv(d);
4084d97c97baSMarc Zyngier }
4085d97c97baSMarc Zyngier
its_vpe_4_1_unmask_irq(struct irq_data * d)4086d97c97baSMarc Zyngier static void its_vpe_4_1_unmask_irq(struct irq_data *d)
4087d97c97baSMarc Zyngier {
4088d97c97baSMarc Zyngier lpi_write_config(d->parent_data, 0, LPI_PROP_ENABLED);
4089d97c97baSMarc Zyngier its_vpe_4_1_send_inv(d);
4090d97c97baSMarc Zyngier }
4091d97c97baSMarc Zyngier
its_vpe_4_1_schedule(struct its_vpe * vpe,struct its_cmd_info * info)409291bf6395SMarc Zyngier static void its_vpe_4_1_schedule(struct its_vpe *vpe,
409391bf6395SMarc Zyngier struct its_cmd_info *info)
409491bf6395SMarc Zyngier {
409591bf6395SMarc Zyngier void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
409691bf6395SMarc Zyngier u64 val = 0;
409791bf6395SMarc Zyngier
409891bf6395SMarc Zyngier /* Schedule the VPE */
409991bf6395SMarc Zyngier val |= GICR_VPENDBASER_Valid;
410091bf6395SMarc Zyngier val |= info->g0en ? GICR_VPENDBASER_4_1_VGRP0EN : 0;
410191bf6395SMarc Zyngier val |= info->g1en ? GICR_VPENDBASER_4_1_VGRP1EN : 0;
410291bf6395SMarc Zyngier val |= FIELD_PREP(GICR_VPENDBASER_4_1_VPEID, vpe->vpe_id);
410391bf6395SMarc Zyngier
41045186a6ccSZenghui Yu gicr_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER);
410591bf6395SMarc Zyngier }
410691bf6395SMarc Zyngier
its_vpe_4_1_deschedule(struct its_vpe * vpe,struct its_cmd_info * info)4107e64fab1aSMarc Zyngier static void its_vpe_4_1_deschedule(struct its_vpe *vpe,
4108e64fab1aSMarc Zyngier struct its_cmd_info *info)
4109e64fab1aSMarc Zyngier {
4110e64fab1aSMarc Zyngier void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
4111e64fab1aSMarc Zyngier u64 val;
4112e64fab1aSMarc Zyngier
4113e64fab1aSMarc Zyngier if (info->req_db) {
4114a3f574cdSMarc Zyngier unsigned long flags;
4115a3f574cdSMarc Zyngier
4116e64fab1aSMarc Zyngier /*
4117e64fab1aSMarc Zyngier * vPE is going to block: make the vPE non-resident with
4118e64fab1aSMarc Zyngier * PendingLast clear and DB set. The GIC guarantees that if
4119e64fab1aSMarc Zyngier * we read-back PendingLast clear, then a doorbell will be
4120e64fab1aSMarc Zyngier * delivered when an interrupt comes.
4121a3f574cdSMarc Zyngier *
4122a3f574cdSMarc Zyngier * Note the locking to deal with the concurrent update of
4123a3f574cdSMarc Zyngier * pending_last from the doorbell interrupt handler that can
4124a3f574cdSMarc Zyngier * run concurrently.
4125e64fab1aSMarc Zyngier */
4126a3f574cdSMarc Zyngier raw_spin_lock_irqsave(&vpe->vpe_lock, flags);
4127e64fab1aSMarc Zyngier val = its_clear_vpend_valid(vlpi_base,
4128e64fab1aSMarc Zyngier GICR_VPENDBASER_PendingLast,
4129e64fab1aSMarc Zyngier GICR_VPENDBASER_4_1_DB);
4130e64fab1aSMarc Zyngier vpe->pending_last = !!(val & GICR_VPENDBASER_PendingLast);
4131a3f574cdSMarc Zyngier raw_spin_unlock_irqrestore(&vpe->vpe_lock, flags);
4132e64fab1aSMarc Zyngier } else {
4133e64fab1aSMarc Zyngier /*
4134e64fab1aSMarc Zyngier * We're not blocking, so just make the vPE non-resident
4135e64fab1aSMarc Zyngier * with PendingLast set, indicating that we'll be back.
4136e64fab1aSMarc Zyngier */
4137e64fab1aSMarc Zyngier val = its_clear_vpend_valid(vlpi_base,
4138e64fab1aSMarc Zyngier 0,
4139e64fab1aSMarc Zyngier GICR_VPENDBASER_PendingLast);
4140e64fab1aSMarc Zyngier vpe->pending_last = true;
4141e64fab1aSMarc Zyngier }
4142e64fab1aSMarc Zyngier }
4143e64fab1aSMarc Zyngier
its_vpe_4_1_invall(struct its_vpe * vpe)4144b4a4bd0fSMarc Zyngier static void its_vpe_4_1_invall(struct its_vpe *vpe)
4145b4a4bd0fSMarc Zyngier {
4146b4a4bd0fSMarc Zyngier void __iomem *rdbase;
41473af9571cSZenghui Yu unsigned long flags;
4148b4a4bd0fSMarc Zyngier u64 val;
41493af9571cSZenghui Yu int cpu;
4150b4a4bd0fSMarc Zyngier
4151b4a4bd0fSMarc Zyngier val = GICR_INVALLR_V;
4152b4a4bd0fSMarc Zyngier val |= FIELD_PREP(GICR_INVALLR_VPEID, vpe->vpe_id);
4153b4a4bd0fSMarc Zyngier
4154b4a4bd0fSMarc Zyngier /* Target the redistributor this vPE is currently known on */
41553af9571cSZenghui Yu cpu = vpe_to_cpuid_lock(vpe, &flags);
41563af9571cSZenghui Yu raw_spin_lock(&gic_data_rdist_cpu(cpu)->rd_lock);
41573af9571cSZenghui Yu rdbase = per_cpu_ptr(gic_rdists->rdist, cpu)->rd_base;
4158b4a4bd0fSMarc Zyngier gic_write_lpir(val, rdbase + GICR_INVALLR);
4159b978c25fSZenghui Yu
4160b978c25fSZenghui Yu wait_for_syncr(rdbase);
41613af9571cSZenghui Yu raw_spin_unlock(&gic_data_rdist_cpu(cpu)->rd_lock);
41623af9571cSZenghui Yu vpe_to_cpuid_unlock(vpe, flags);
4163b4a4bd0fSMarc Zyngier }
4164b4a4bd0fSMarc Zyngier
its_vpe_4_1_set_vcpu_affinity(struct irq_data * d,void * vcpu_info)416529c647f3SMarc Zyngier static int its_vpe_4_1_set_vcpu_affinity(struct irq_data *d, void *vcpu_info)
416629c647f3SMarc Zyngier {
416791bf6395SMarc Zyngier struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
416829c647f3SMarc Zyngier struct its_cmd_info *info = vcpu_info;
416929c647f3SMarc Zyngier
417029c647f3SMarc Zyngier switch (info->cmd_type) {
417129c647f3SMarc Zyngier case SCHEDULE_VPE:
417291bf6395SMarc Zyngier its_vpe_4_1_schedule(vpe, info);
417329c647f3SMarc Zyngier return 0;
417429c647f3SMarc Zyngier
417529c647f3SMarc Zyngier case DESCHEDULE_VPE:
4176e64fab1aSMarc Zyngier its_vpe_4_1_deschedule(vpe, info);
417729c647f3SMarc Zyngier return 0;
417829c647f3SMarc Zyngier
417957e3cebdSShenming Lu case COMMIT_VPE:
418057e3cebdSShenming Lu its_wait_vpt_parse_complete();
418157e3cebdSShenming Lu return 0;
418257e3cebdSShenming Lu
418329c647f3SMarc Zyngier case INVALL_VPE:
4184b4a4bd0fSMarc Zyngier its_vpe_4_1_invall(vpe);
418529c647f3SMarc Zyngier return 0;
418629c647f3SMarc Zyngier
418729c647f3SMarc Zyngier default:
418829c647f3SMarc Zyngier return -EINVAL;
418929c647f3SMarc Zyngier }
419029c647f3SMarc Zyngier }
419129c647f3SMarc Zyngier
419229c647f3SMarc Zyngier static struct irq_chip its_vpe_4_1_irq_chip = {
419329c647f3SMarc Zyngier .name = "GICv4.1-vpe",
4194d97c97baSMarc Zyngier .irq_mask = its_vpe_4_1_mask_irq,
4195d97c97baSMarc Zyngier .irq_unmask = its_vpe_4_1_unmask_irq,
419629c647f3SMarc Zyngier .irq_eoi = irq_chip_eoi_parent,
419729c647f3SMarc Zyngier .irq_set_affinity = its_vpe_set_affinity,
419829c647f3SMarc Zyngier .irq_set_vcpu_affinity = its_vpe_4_1_set_vcpu_affinity,
419929c647f3SMarc Zyngier };
420029c647f3SMarc Zyngier
its_configure_sgi(struct irq_data * d,bool clear)4201e252cf8aSMarc Zyngier static void its_configure_sgi(struct irq_data *d, bool clear)
4202e252cf8aSMarc Zyngier {
4203e252cf8aSMarc Zyngier struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4204e252cf8aSMarc Zyngier struct its_cmd_desc desc;
4205e252cf8aSMarc Zyngier
4206e252cf8aSMarc Zyngier desc.its_vsgi_cmd.vpe = vpe;
4207e252cf8aSMarc Zyngier desc.its_vsgi_cmd.sgi = d->hwirq;
4208e252cf8aSMarc Zyngier desc.its_vsgi_cmd.priority = vpe->sgi_config[d->hwirq].priority;
4209e252cf8aSMarc Zyngier desc.its_vsgi_cmd.enable = vpe->sgi_config[d->hwirq].enabled;
4210e252cf8aSMarc Zyngier desc.its_vsgi_cmd.group = vpe->sgi_config[d->hwirq].group;
4211e252cf8aSMarc Zyngier desc.its_vsgi_cmd.clear = clear;
4212e252cf8aSMarc Zyngier
4213e252cf8aSMarc Zyngier /*
4214e252cf8aSMarc Zyngier * GICv4.1 allows us to send VSGI commands to any ITS as long as the
4215e252cf8aSMarc Zyngier * destination VPE is mapped there. Since we map them eagerly at
4216e252cf8aSMarc Zyngier * activation time, we're pretty sure the first GICv4.1 ITS will do.
4217e252cf8aSMarc Zyngier */
4218e252cf8aSMarc Zyngier its_send_single_vcommand(find_4_1_its(), its_build_vsgi_cmd, &desc);
4219e252cf8aSMarc Zyngier }
4220e252cf8aSMarc Zyngier
its_sgi_mask_irq(struct irq_data * d)4221b4e8d644SMarc Zyngier static void its_sgi_mask_irq(struct irq_data *d)
4222b4e8d644SMarc Zyngier {
4223b4e8d644SMarc Zyngier struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4224b4e8d644SMarc Zyngier
4225b4e8d644SMarc Zyngier vpe->sgi_config[d->hwirq].enabled = false;
4226b4e8d644SMarc Zyngier its_configure_sgi(d, false);
4227b4e8d644SMarc Zyngier }
4228b4e8d644SMarc Zyngier
its_sgi_unmask_irq(struct irq_data * d)4229b4e8d644SMarc Zyngier static void its_sgi_unmask_irq(struct irq_data *d)
4230b4e8d644SMarc Zyngier {
4231b4e8d644SMarc Zyngier struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4232b4e8d644SMarc Zyngier
4233b4e8d644SMarc Zyngier vpe->sgi_config[d->hwirq].enabled = true;
4234b4e8d644SMarc Zyngier its_configure_sgi(d, false);
4235b4e8d644SMarc Zyngier }
4236b4e8d644SMarc Zyngier
its_sgi_set_affinity(struct irq_data * d,const struct cpumask * mask_val,bool force)4237166cba71SMarc Zyngier static int its_sgi_set_affinity(struct irq_data *d,
4238166cba71SMarc Zyngier const struct cpumask *mask_val,
4239166cba71SMarc Zyngier bool force)
4240166cba71SMarc Zyngier {
4241166cba71SMarc Zyngier /*
4242166cba71SMarc Zyngier * There is no notion of affinity for virtual SGIs, at least
4243a359f757SIngo Molnar * not on the host (since they can only be targeting a vPE).
4244166cba71SMarc Zyngier * Tell the kernel we've done whatever it asked for.
4245166cba71SMarc Zyngier */
42464b2dfe1eSMarc Zyngier irq_data_update_effective_affinity(d, mask_val);
4247166cba71SMarc Zyngier return IRQ_SET_MASK_OK;
4248166cba71SMarc Zyngier }
4249166cba71SMarc Zyngier
its_sgi_set_irqchip_state(struct irq_data * d,enum irqchip_irq_state which,bool state)42507017ff0eSMarc Zyngier static int its_sgi_set_irqchip_state(struct irq_data *d,
42517017ff0eSMarc Zyngier enum irqchip_irq_state which,
42527017ff0eSMarc Zyngier bool state)
42537017ff0eSMarc Zyngier {
42547017ff0eSMarc Zyngier if (which != IRQCHIP_STATE_PENDING)
42557017ff0eSMarc Zyngier return -EINVAL;
42567017ff0eSMarc Zyngier
42577017ff0eSMarc Zyngier if (state) {
42587017ff0eSMarc Zyngier struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
42597017ff0eSMarc Zyngier struct its_node *its = find_4_1_its();
42607017ff0eSMarc Zyngier u64 val;
42617017ff0eSMarc Zyngier
42627017ff0eSMarc Zyngier val = FIELD_PREP(GITS_SGIR_VPEID, vpe->vpe_id);
42637017ff0eSMarc Zyngier val |= FIELD_PREP(GITS_SGIR_VINTID, d->hwirq);
42647017ff0eSMarc Zyngier writeq_relaxed(val, its->sgir_base + GITS_SGIR - SZ_128K);
42657017ff0eSMarc Zyngier } else {
42667017ff0eSMarc Zyngier its_configure_sgi(d, true);
42677017ff0eSMarc Zyngier }
42687017ff0eSMarc Zyngier
42697017ff0eSMarc Zyngier return 0;
42707017ff0eSMarc Zyngier }
42717017ff0eSMarc Zyngier
its_sgi_get_irqchip_state(struct irq_data * d,enum irqchip_irq_state which,bool * val)42727017ff0eSMarc Zyngier static int its_sgi_get_irqchip_state(struct irq_data *d,
42737017ff0eSMarc Zyngier enum irqchip_irq_state which, bool *val)
42747017ff0eSMarc Zyngier {
42757017ff0eSMarc Zyngier struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
42767017ff0eSMarc Zyngier void __iomem *base;
42777017ff0eSMarc Zyngier unsigned long flags;
42787017ff0eSMarc Zyngier u32 count = 1000000; /* 1s! */
42797017ff0eSMarc Zyngier u32 status;
42807017ff0eSMarc Zyngier int cpu;
42817017ff0eSMarc Zyngier
42827017ff0eSMarc Zyngier if (which != IRQCHIP_STATE_PENDING)
42837017ff0eSMarc Zyngier return -EINVAL;
42847017ff0eSMarc Zyngier
42857017ff0eSMarc Zyngier /*
42867017ff0eSMarc Zyngier * Locking galore! We can race against two different events:
42877017ff0eSMarc Zyngier *
4288a359f757SIngo Molnar * - Concurrent vPE affinity change: we must make sure it cannot
42897017ff0eSMarc Zyngier * happen, or we'll talk to the wrong redistributor. This is
42907017ff0eSMarc Zyngier * identical to what happens with vLPIs.
42917017ff0eSMarc Zyngier *
42927017ff0eSMarc Zyngier * - Concurrent VSGIPENDR access: As it involves accessing two
42937017ff0eSMarc Zyngier * MMIO registers, this must be made atomic one way or another.
42947017ff0eSMarc Zyngier */
42957017ff0eSMarc Zyngier cpu = vpe_to_cpuid_lock(vpe, &flags);
42967017ff0eSMarc Zyngier raw_spin_lock(&gic_data_rdist_cpu(cpu)->rd_lock);
42977017ff0eSMarc Zyngier base = gic_data_rdist_cpu(cpu)->rd_base + SZ_128K;
42987017ff0eSMarc Zyngier writel_relaxed(vpe->vpe_id, base + GICR_VSGIR);
42997017ff0eSMarc Zyngier do {
43007017ff0eSMarc Zyngier status = readl_relaxed(base + GICR_VSGIPENDR);
43017017ff0eSMarc Zyngier if (!(status & GICR_VSGIPENDR_BUSY))
43027017ff0eSMarc Zyngier goto out;
43037017ff0eSMarc Zyngier
43047017ff0eSMarc Zyngier count--;
43057017ff0eSMarc Zyngier if (!count) {
43067017ff0eSMarc Zyngier pr_err_ratelimited("Unable to get SGI status\n");
43077017ff0eSMarc Zyngier goto out;
43087017ff0eSMarc Zyngier }
43097017ff0eSMarc Zyngier cpu_relax();
43107017ff0eSMarc Zyngier udelay(1);
43117017ff0eSMarc Zyngier } while (count);
43127017ff0eSMarc Zyngier
43137017ff0eSMarc Zyngier out:
43147017ff0eSMarc Zyngier raw_spin_unlock(&gic_data_rdist_cpu(cpu)->rd_lock);
43157017ff0eSMarc Zyngier vpe_to_cpuid_unlock(vpe, flags);
43167017ff0eSMarc Zyngier
43177017ff0eSMarc Zyngier if (!count)
43187017ff0eSMarc Zyngier return -ENXIO;
43197017ff0eSMarc Zyngier
43207017ff0eSMarc Zyngier *val = !!(status & (1 << d->hwirq));
43217017ff0eSMarc Zyngier
43227017ff0eSMarc Zyngier return 0;
43237017ff0eSMarc Zyngier }
43247017ff0eSMarc Zyngier
its_sgi_set_vcpu_affinity(struct irq_data * d,void * vcpu_info)432505d32df1SMarc Zyngier static int its_sgi_set_vcpu_affinity(struct irq_data *d, void *vcpu_info)
432605d32df1SMarc Zyngier {
432705d32df1SMarc Zyngier struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
432805d32df1SMarc Zyngier struct its_cmd_info *info = vcpu_info;
432905d32df1SMarc Zyngier
433005d32df1SMarc Zyngier switch (info->cmd_type) {
433105d32df1SMarc Zyngier case PROP_UPDATE_VSGI:
433205d32df1SMarc Zyngier vpe->sgi_config[d->hwirq].priority = info->priority;
433305d32df1SMarc Zyngier vpe->sgi_config[d->hwirq].group = info->group;
433405d32df1SMarc Zyngier its_configure_sgi(d, false);
433505d32df1SMarc Zyngier return 0;
433605d32df1SMarc Zyngier
433705d32df1SMarc Zyngier default:
433805d32df1SMarc Zyngier return -EINVAL;
433905d32df1SMarc Zyngier }
434005d32df1SMarc Zyngier }
434105d32df1SMarc Zyngier
4342166cba71SMarc Zyngier static struct irq_chip its_sgi_irq_chip = {
4343166cba71SMarc Zyngier .name = "GICv4.1-sgi",
4344b4e8d644SMarc Zyngier .irq_mask = its_sgi_mask_irq,
4345b4e8d644SMarc Zyngier .irq_unmask = its_sgi_unmask_irq,
4346166cba71SMarc Zyngier .irq_set_affinity = its_sgi_set_affinity,
43477017ff0eSMarc Zyngier .irq_set_irqchip_state = its_sgi_set_irqchip_state,
43487017ff0eSMarc Zyngier .irq_get_irqchip_state = its_sgi_get_irqchip_state,
434905d32df1SMarc Zyngier .irq_set_vcpu_affinity = its_sgi_set_vcpu_affinity,
4350166cba71SMarc Zyngier };
4351166cba71SMarc Zyngier
its_sgi_irq_domain_alloc(struct irq_domain * domain,unsigned int virq,unsigned int nr_irqs,void * args)4352166cba71SMarc Zyngier static int its_sgi_irq_domain_alloc(struct irq_domain *domain,
4353166cba71SMarc Zyngier unsigned int virq, unsigned int nr_irqs,
4354166cba71SMarc Zyngier void *args)
4355166cba71SMarc Zyngier {
4356166cba71SMarc Zyngier struct its_vpe *vpe = args;
4357166cba71SMarc Zyngier int i;
4358166cba71SMarc Zyngier
4359166cba71SMarc Zyngier /* Yes, we do want 16 SGIs */
4360166cba71SMarc Zyngier WARN_ON(nr_irqs != 16);
4361166cba71SMarc Zyngier
4362166cba71SMarc Zyngier for (i = 0; i < 16; i++) {
4363166cba71SMarc Zyngier vpe->sgi_config[i].priority = 0;
4364166cba71SMarc Zyngier vpe->sgi_config[i].enabled = false;
4365166cba71SMarc Zyngier vpe->sgi_config[i].group = false;
4366166cba71SMarc Zyngier
4367166cba71SMarc Zyngier irq_domain_set_hwirq_and_chip(domain, virq + i, i,
4368166cba71SMarc Zyngier &its_sgi_irq_chip, vpe);
4369166cba71SMarc Zyngier irq_set_status_flags(virq + i, IRQ_DISABLE_UNLAZY);
4370166cba71SMarc Zyngier }
4371166cba71SMarc Zyngier
4372166cba71SMarc Zyngier return 0;
4373166cba71SMarc Zyngier }
4374166cba71SMarc Zyngier
its_sgi_irq_domain_free(struct irq_domain * domain,unsigned int virq,unsigned int nr_irqs)4375166cba71SMarc Zyngier static void its_sgi_irq_domain_free(struct irq_domain *domain,
4376166cba71SMarc Zyngier unsigned int virq,
4377166cba71SMarc Zyngier unsigned int nr_irqs)
4378166cba71SMarc Zyngier {
4379166cba71SMarc Zyngier /* Nothing to do */
4380166cba71SMarc Zyngier }
4381166cba71SMarc Zyngier
its_sgi_irq_domain_activate(struct irq_domain * domain,struct irq_data * d,bool reserve)4382166cba71SMarc Zyngier static int its_sgi_irq_domain_activate(struct irq_domain *domain,
4383166cba71SMarc Zyngier struct irq_data *d, bool reserve)
4384166cba71SMarc Zyngier {
4385e252cf8aSMarc Zyngier /* Write out the initial SGI configuration */
4386e252cf8aSMarc Zyngier its_configure_sgi(d, false);
4387166cba71SMarc Zyngier return 0;
4388166cba71SMarc Zyngier }
4389166cba71SMarc Zyngier
its_sgi_irq_domain_deactivate(struct irq_domain * domain,struct irq_data * d)4390166cba71SMarc Zyngier static void its_sgi_irq_domain_deactivate(struct irq_domain *domain,
4391166cba71SMarc Zyngier struct irq_data *d)
4392166cba71SMarc Zyngier {
4393e252cf8aSMarc Zyngier struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4394e252cf8aSMarc Zyngier
4395e252cf8aSMarc Zyngier /*
4396e252cf8aSMarc Zyngier * The VSGI command is awkward:
4397e252cf8aSMarc Zyngier *
4398e252cf8aSMarc Zyngier * - To change the configuration, CLEAR must be set to false,
4399e252cf8aSMarc Zyngier * leaving the pending bit unchanged.
4400e252cf8aSMarc Zyngier * - To clear the pending bit, CLEAR must be set to true, leaving
4401e252cf8aSMarc Zyngier * the configuration unchanged.
4402e252cf8aSMarc Zyngier *
4403e252cf8aSMarc Zyngier * You just can't do both at once, hence the two commands below.
4404e252cf8aSMarc Zyngier */
4405e252cf8aSMarc Zyngier vpe->sgi_config[d->hwirq].enabled = false;
4406e252cf8aSMarc Zyngier its_configure_sgi(d, false);
4407e252cf8aSMarc Zyngier its_configure_sgi(d, true);
4408166cba71SMarc Zyngier }
4409166cba71SMarc Zyngier
4410166cba71SMarc Zyngier static const struct irq_domain_ops its_sgi_domain_ops = {
4411166cba71SMarc Zyngier .alloc = its_sgi_irq_domain_alloc,
4412166cba71SMarc Zyngier .free = its_sgi_irq_domain_free,
4413166cba71SMarc Zyngier .activate = its_sgi_irq_domain_activate,
4414166cba71SMarc Zyngier .deactivate = its_sgi_irq_domain_deactivate,
4415166cba71SMarc Zyngier };
4416166cba71SMarc Zyngier
its_vpe_id_alloc(void)44177d75bbb4SMarc Zyngier static int its_vpe_id_alloc(void)
44187d75bbb4SMarc Zyngier {
441932bd44dcSShanker Donthineni return ida_simple_get(&its_vpeid_ida, 0, ITS_MAX_VPEID, GFP_KERNEL);
44207d75bbb4SMarc Zyngier }
44217d75bbb4SMarc Zyngier
its_vpe_id_free(u16 id)44227d75bbb4SMarc Zyngier static void its_vpe_id_free(u16 id)
44237d75bbb4SMarc Zyngier {
44247d75bbb4SMarc Zyngier ida_simple_remove(&its_vpeid_ida, id);
44257d75bbb4SMarc Zyngier }
44267d75bbb4SMarc Zyngier
its_vpe_init(struct its_vpe * vpe)44277d75bbb4SMarc Zyngier static int its_vpe_init(struct its_vpe *vpe)
44287d75bbb4SMarc Zyngier {
44297d75bbb4SMarc Zyngier struct page *vpt_page;
44307d75bbb4SMarc Zyngier int vpe_id;
44317d75bbb4SMarc Zyngier
44327d75bbb4SMarc Zyngier /* Allocate vpe_id */
44337d75bbb4SMarc Zyngier vpe_id = its_vpe_id_alloc();
44347d75bbb4SMarc Zyngier if (vpe_id < 0)
44357d75bbb4SMarc Zyngier return vpe_id;
44367d75bbb4SMarc Zyngier
44377d75bbb4SMarc Zyngier /* Allocate VPT */
44387d75bbb4SMarc Zyngier vpt_page = its_allocate_pending_table(GFP_KERNEL);
44397d75bbb4SMarc Zyngier if (!vpt_page) {
44407d75bbb4SMarc Zyngier its_vpe_id_free(vpe_id);
44417d75bbb4SMarc Zyngier return -ENOMEM;
44427d75bbb4SMarc Zyngier }
44437d75bbb4SMarc Zyngier
44447d75bbb4SMarc Zyngier if (!its_alloc_vpe_table(vpe_id)) {
44457d75bbb4SMarc Zyngier its_vpe_id_free(vpe_id);
444634f8eb92SNianyao Tang its_free_pending_table(vpt_page);
44477d75bbb4SMarc Zyngier return -ENOMEM;
44487d75bbb4SMarc Zyngier }
44497d75bbb4SMarc Zyngier
4450f3a05921SMarc Zyngier raw_spin_lock_init(&vpe->vpe_lock);
44517d75bbb4SMarc Zyngier vpe->vpe_id = vpe_id;
44527d75bbb4SMarc Zyngier vpe->vpt_page = vpt_page;
445364edfaa9SMarc Zyngier if (gic_rdists->has_rvpeid)
445464edfaa9SMarc Zyngier atomic_set(&vpe->vmapp_count, 0);
445564edfaa9SMarc Zyngier else
445620b3d54eSMarc Zyngier vpe->vpe_proxy_event = -1;
44577d75bbb4SMarc Zyngier
44587d75bbb4SMarc Zyngier return 0;
44597d75bbb4SMarc Zyngier }
44607d75bbb4SMarc Zyngier
its_vpe_teardown(struct its_vpe * vpe)44617d75bbb4SMarc Zyngier static void its_vpe_teardown(struct its_vpe *vpe)
44627d75bbb4SMarc Zyngier {
446320b3d54eSMarc Zyngier its_vpe_db_proxy_unmap(vpe);
44647d75bbb4SMarc Zyngier its_vpe_id_free(vpe->vpe_id);
44657d75bbb4SMarc Zyngier its_free_pending_table(vpe->vpt_page);
44667d75bbb4SMarc Zyngier }
44677d75bbb4SMarc Zyngier
its_vpe_irq_domain_free(struct irq_domain * domain,unsigned int virq,unsigned int nr_irqs)44687d75bbb4SMarc Zyngier static void its_vpe_irq_domain_free(struct irq_domain *domain,
44697d75bbb4SMarc Zyngier unsigned int virq,
44707d75bbb4SMarc Zyngier unsigned int nr_irqs)
44717d75bbb4SMarc Zyngier {
44727d75bbb4SMarc Zyngier struct its_vm *vm = domain->host_data;
44737d75bbb4SMarc Zyngier int i;
44747d75bbb4SMarc Zyngier
44757d75bbb4SMarc Zyngier irq_domain_free_irqs_parent(domain, virq, nr_irqs);
44767d75bbb4SMarc Zyngier
44777d75bbb4SMarc Zyngier for (i = 0; i < nr_irqs; i++) {
44787d75bbb4SMarc Zyngier struct irq_data *data = irq_domain_get_irq_data(domain,
44797d75bbb4SMarc Zyngier virq + i);
44807d75bbb4SMarc Zyngier struct its_vpe *vpe = irq_data_get_irq_chip_data(data);
44817d75bbb4SMarc Zyngier
44827d75bbb4SMarc Zyngier BUG_ON(vm != vpe->its_vm);
44837d75bbb4SMarc Zyngier
44847d75bbb4SMarc Zyngier clear_bit(data->hwirq, vm->db_bitmap);
44857d75bbb4SMarc Zyngier its_vpe_teardown(vpe);
44867d75bbb4SMarc Zyngier irq_domain_reset_irq_data(data);
44877d75bbb4SMarc Zyngier }
44887d75bbb4SMarc Zyngier
44897d75bbb4SMarc Zyngier if (bitmap_empty(vm->db_bitmap, vm->nr_db_lpis)) {
449038dd7c49SMarc Zyngier its_lpi_free(vm->db_bitmap, vm->db_lpi_base, vm->nr_db_lpis);
44917d75bbb4SMarc Zyngier its_free_prop_table(vm->vprop_page);
44927d75bbb4SMarc Zyngier }
44937d75bbb4SMarc Zyngier }
44947d75bbb4SMarc Zyngier
its_vpe_irq_domain_alloc(struct irq_domain * domain,unsigned int virq,unsigned int nr_irqs,void * args)44957d75bbb4SMarc Zyngier static int its_vpe_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
44967d75bbb4SMarc Zyngier unsigned int nr_irqs, void *args)
44977d75bbb4SMarc Zyngier {
449829c647f3SMarc Zyngier struct irq_chip *irqchip = &its_vpe_irq_chip;
44997d75bbb4SMarc Zyngier struct its_vm *vm = args;
45007d75bbb4SMarc Zyngier unsigned long *bitmap;
45017d75bbb4SMarc Zyngier struct page *vprop_page;
45027d75bbb4SMarc Zyngier int base, nr_ids, i, err = 0;
45037d75bbb4SMarc Zyngier
45047d75bbb4SMarc Zyngier BUG_ON(!vm);
45057d75bbb4SMarc Zyngier
450638dd7c49SMarc Zyngier bitmap = its_lpi_alloc(roundup_pow_of_two(nr_irqs), &base, &nr_ids);
45077d75bbb4SMarc Zyngier if (!bitmap)
45087d75bbb4SMarc Zyngier return -ENOMEM;
45097d75bbb4SMarc Zyngier
45107d75bbb4SMarc Zyngier if (nr_ids < nr_irqs) {
451138dd7c49SMarc Zyngier its_lpi_free(bitmap, base, nr_ids);
45127d75bbb4SMarc Zyngier return -ENOMEM;
45137d75bbb4SMarc Zyngier }
45147d75bbb4SMarc Zyngier
45157d75bbb4SMarc Zyngier vprop_page = its_allocate_prop_table(GFP_KERNEL);
45167d75bbb4SMarc Zyngier if (!vprop_page) {
451738dd7c49SMarc Zyngier its_lpi_free(bitmap, base, nr_ids);
45187d75bbb4SMarc Zyngier return -ENOMEM;
45197d75bbb4SMarc Zyngier }
45207d75bbb4SMarc Zyngier
45217d75bbb4SMarc Zyngier vm->db_bitmap = bitmap;
45227d75bbb4SMarc Zyngier vm->db_lpi_base = base;
45237d75bbb4SMarc Zyngier vm->nr_db_lpis = nr_ids;
45247d75bbb4SMarc Zyngier vm->vprop_page = vprop_page;
45257d75bbb4SMarc Zyngier
452629c647f3SMarc Zyngier if (gic_rdists->has_rvpeid)
452729c647f3SMarc Zyngier irqchip = &its_vpe_4_1_irq_chip;
452829c647f3SMarc Zyngier
45297d75bbb4SMarc Zyngier for (i = 0; i < nr_irqs; i++) {
45307d75bbb4SMarc Zyngier vm->vpes[i]->vpe_db_lpi = base + i;
45317d75bbb4SMarc Zyngier err = its_vpe_init(vm->vpes[i]);
45327d75bbb4SMarc Zyngier if (err)
45337d75bbb4SMarc Zyngier break;
45347d75bbb4SMarc Zyngier err = its_irq_gic_domain_alloc(domain, virq + i,
45357d75bbb4SMarc Zyngier vm->vpes[i]->vpe_db_lpi);
45367d75bbb4SMarc Zyngier if (err)
45377d75bbb4SMarc Zyngier break;
45387d75bbb4SMarc Zyngier irq_domain_set_hwirq_and_chip(domain, virq + i, i,
453929c647f3SMarc Zyngier irqchip, vm->vpes[i]);
45407d75bbb4SMarc Zyngier set_bit(i, bitmap);
45418f4b5895SJames Gowans irqd_set_resend_when_in_progress(irq_get_irq_data(virq + i));
45427d75bbb4SMarc Zyngier }
45437d75bbb4SMarc Zyngier
454403170e65SGuanrui Huang if (err)
4545280bef51SKaige Fu its_vpe_irq_domain_free(domain, virq, i);
45467d75bbb4SMarc Zyngier
45477d75bbb4SMarc Zyngier return err;
45487d75bbb4SMarc Zyngier }
45497d75bbb4SMarc Zyngier
its_vpe_irq_domain_activate(struct irq_domain * domain,struct irq_data * d,bool reserve)455072491643SThomas Gleixner static int its_vpe_irq_domain_activate(struct irq_domain *domain,
4551702cb0a0SThomas Gleixner struct irq_data *d, bool reserve)
4552eb78192bSMarc Zyngier {
4553eb78192bSMarc Zyngier struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
455440619a2eSMarc Zyngier struct its_node *its;
4555eb78192bSMarc Zyngier
4556009384b3SMarc Zyngier /*
4557009384b3SMarc Zyngier * If we use the list map, we issue VMAPP on demand... Unless
4558009384b3SMarc Zyngier * we're on a GICv4.1 and we eagerly map the VPE on all ITSs
4559009384b3SMarc Zyngier * so that VSGIs can work.
4560009384b3SMarc Zyngier */
4561009384b3SMarc Zyngier if (!gic_requires_eager_mapping())
45626ef930f2SMarc Zyngier return 0;
4563eb78192bSMarc Zyngier
4564eb78192bSMarc Zyngier /* Map the VPE to the first possible CPU */
4565eb78192bSMarc Zyngier vpe->col_idx = cpumask_first(cpu_online_mask);
456640619a2eSMarc Zyngier
456740619a2eSMarc Zyngier list_for_each_entry(its, &its_nodes, entry) {
45680dd57fedSMarc Zyngier if (!is_v4(its))
456940619a2eSMarc Zyngier continue;
457040619a2eSMarc Zyngier
457175fd951bSMarc Zyngier its_send_vmapp(its, vpe, true);
457240619a2eSMarc Zyngier its_send_vinvall(its, vpe);
457340619a2eSMarc Zyngier }
457440619a2eSMarc Zyngier
457544c4c25eSMarc Zyngier irq_data_update_effective_affinity(d, cpumask_of(vpe->col_idx));
457644c4c25eSMarc Zyngier
457772491643SThomas Gleixner return 0;
4578eb78192bSMarc Zyngier }
4579eb78192bSMarc Zyngier
its_vpe_irq_domain_deactivate(struct irq_domain * domain,struct irq_data * d)4580eb78192bSMarc Zyngier static void its_vpe_irq_domain_deactivate(struct irq_domain *domain,
4581eb78192bSMarc Zyngier struct irq_data *d)
4582eb78192bSMarc Zyngier {
4583eb78192bSMarc Zyngier struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
458475fd951bSMarc Zyngier struct its_node *its;
4585eb78192bSMarc Zyngier
45862247e1bfSMarc Zyngier /*
4587009384b3SMarc Zyngier * If we use the list map on GICv4.0, we unmap the VPE once no
4588009384b3SMarc Zyngier * VLPIs are associated with the VM.
45892247e1bfSMarc Zyngier */
4590009384b3SMarc Zyngier if (!gic_requires_eager_mapping())
45912247e1bfSMarc Zyngier return;
45922247e1bfSMarc Zyngier
459375fd951bSMarc Zyngier list_for_each_entry(its, &its_nodes, entry) {
45940dd57fedSMarc Zyngier if (!is_v4(its))
459575fd951bSMarc Zyngier continue;
459675fd951bSMarc Zyngier
459775fd951bSMarc Zyngier its_send_vmapp(its, vpe, false);
459875fd951bSMarc Zyngier }
4599301beaf1SMarc Zyngier
4600301beaf1SMarc Zyngier /*
4601301beaf1SMarc Zyngier * There may be a direct read to the VPT after unmapping the
4602301beaf1SMarc Zyngier * vPE, to guarantee the validity of this, we make the VPT
4603301beaf1SMarc Zyngier * memory coherent with the CPU caches here.
4604301beaf1SMarc Zyngier */
4605301beaf1SMarc Zyngier if (find_4_1_its() && !atomic_read(&vpe->vmapp_count))
4606301beaf1SMarc Zyngier gic_flush_dcache_to_poc(page_address(vpe->vpt_page),
4607301beaf1SMarc Zyngier LPI_PENDBASE_SZ);
4608eb78192bSMarc Zyngier }
4609eb78192bSMarc Zyngier
46108fff27aeSMarc Zyngier static const struct irq_domain_ops its_vpe_domain_ops = {
46117d75bbb4SMarc Zyngier .alloc = its_vpe_irq_domain_alloc,
46127d75bbb4SMarc Zyngier .free = its_vpe_irq_domain_free,
4613eb78192bSMarc Zyngier .activate = its_vpe_irq_domain_activate,
4614eb78192bSMarc Zyngier .deactivate = its_vpe_irq_domain_deactivate,
46158fff27aeSMarc Zyngier };
46168fff27aeSMarc Zyngier
its_force_quiescent(void __iomem * base)46174559fbb3SYun Wu static int its_force_quiescent(void __iomem *base)
46184559fbb3SYun Wu {
46194559fbb3SYun Wu u32 count = 1000000; /* 1s */
46204559fbb3SYun Wu u32 val;
46214559fbb3SYun Wu
46224559fbb3SYun Wu val = readl_relaxed(base + GITS_CTLR);
46237611da86SDavid Daney /*
46247611da86SDavid Daney * GIC architecture specification requires the ITS to be both
46257611da86SDavid Daney * disabled and quiescent for writes to GITS_BASER<n> or
46267611da86SDavid Daney * GITS_CBASER to not have UNPREDICTABLE results.
46277611da86SDavid Daney */
46287611da86SDavid Daney if ((val & GITS_CTLR_QUIESCENT) && !(val & GITS_CTLR_ENABLE))
46294559fbb3SYun Wu return 0;
46304559fbb3SYun Wu
46314559fbb3SYun Wu /* Disable the generation of all interrupts to this ITS */
4632d51c4b4dSMarc Zyngier val &= ~(GITS_CTLR_ENABLE | GITS_CTLR_ImDe);
46334559fbb3SYun Wu writel_relaxed(val, base + GITS_CTLR);
46344559fbb3SYun Wu
46354559fbb3SYun Wu /* Poll GITS_CTLR and wait until ITS becomes quiescent */
46364559fbb3SYun Wu while (1) {
46374559fbb3SYun Wu val = readl_relaxed(base + GITS_CTLR);
46384559fbb3SYun Wu if (val & GITS_CTLR_QUIESCENT)
46394559fbb3SYun Wu return 0;
46404559fbb3SYun Wu
46414559fbb3SYun Wu count--;
46424559fbb3SYun Wu if (!count)
46434559fbb3SYun Wu return -EBUSY;
46444559fbb3SYun Wu
46454559fbb3SYun Wu cpu_relax();
46464559fbb3SYun Wu udelay(1);
46474559fbb3SYun Wu }
46484559fbb3SYun Wu }
46494559fbb3SYun Wu
its_enable_quirk_cavium_22375(void * data)46509d111d49SArd Biesheuvel static bool __maybe_unused its_enable_quirk_cavium_22375(void *data)
465194100970SRobert Richter {
465294100970SRobert Richter struct its_node *its = data;
465394100970SRobert Richter
4654576a8342SMarc Zyngier /* erratum 22375: only alloc 8MB table size (20 bits) */
4655576a8342SMarc Zyngier its->typer &= ~GITS_TYPER_DEVBITS;
4656576a8342SMarc Zyngier its->typer |= FIELD_PREP(GITS_TYPER_DEVBITS, 20 - 1);
465794100970SRobert Richter its->flags |= ITS_FLAGS_WORKAROUND_CAVIUM_22375;
46589d111d49SArd Biesheuvel
46599d111d49SArd Biesheuvel return true;
466094100970SRobert Richter }
466194100970SRobert Richter
its_enable_quirk_cavium_23144(void * data)46629d111d49SArd Biesheuvel static bool __maybe_unused its_enable_quirk_cavium_23144(void *data)
4663fbf8f40eSGanapatrao Kulkarni {
4664fbf8f40eSGanapatrao Kulkarni struct its_node *its = data;
4665fbf8f40eSGanapatrao Kulkarni
4666fbf8f40eSGanapatrao Kulkarni its->flags |= ITS_FLAGS_WORKAROUND_CAVIUM_23144;
46679d111d49SArd Biesheuvel
46689d111d49SArd Biesheuvel return true;
4669fbf8f40eSGanapatrao Kulkarni }
4670fbf8f40eSGanapatrao Kulkarni
its_enable_quirk_qdf2400_e0065(void * data)46719d111d49SArd Biesheuvel static bool __maybe_unused its_enable_quirk_qdf2400_e0065(void *data)
467290922a2dSShanker Donthineni {
467390922a2dSShanker Donthineni struct its_node *its = data;
467490922a2dSShanker Donthineni
467590922a2dSShanker Donthineni /* On QDF2400, the size of the ITE is 16Bytes */
4676ffedbf0cSMarc Zyngier its->typer &= ~GITS_TYPER_ITT_ENTRY_SIZE;
4677ffedbf0cSMarc Zyngier its->typer |= FIELD_PREP(GITS_TYPER_ITT_ENTRY_SIZE, 16 - 1);
46789d111d49SArd Biesheuvel
46799d111d49SArd Biesheuvel return true;
468090922a2dSShanker Donthineni }
468190922a2dSShanker Donthineni
its_irq_get_msi_base_pre_its(struct its_device * its_dev)4682558b0165SArd Biesheuvel static u64 its_irq_get_msi_base_pre_its(struct its_device *its_dev)
4683558b0165SArd Biesheuvel {
4684558b0165SArd Biesheuvel struct its_node *its = its_dev->its;
4685558b0165SArd Biesheuvel
4686558b0165SArd Biesheuvel /*
4687558b0165SArd Biesheuvel * The Socionext Synquacer SoC has a so-called 'pre-ITS',
4688558b0165SArd Biesheuvel * which maps 32-bit writes targeted at a separate window of
4689558b0165SArd Biesheuvel * size '4 << device_id_bits' onto writes to GITS_TRANSLATER
4690558b0165SArd Biesheuvel * with device ID taken from bits [device_id_bits + 1:2] of
4691558b0165SArd Biesheuvel * the window offset.
4692558b0165SArd Biesheuvel */
4693558b0165SArd Biesheuvel return its->pre_its_base + (its_dev->device_id << 2);
4694558b0165SArd Biesheuvel }
4695558b0165SArd Biesheuvel
its_enable_quirk_socionext_synquacer(void * data)4696558b0165SArd Biesheuvel static bool __maybe_unused its_enable_quirk_socionext_synquacer(void *data)
4697558b0165SArd Biesheuvel {
4698558b0165SArd Biesheuvel struct its_node *its = data;
4699558b0165SArd Biesheuvel u32 pre_its_window[2];
4700558b0165SArd Biesheuvel u32 ids;
4701558b0165SArd Biesheuvel
4702558b0165SArd Biesheuvel if (!fwnode_property_read_u32_array(its->fwnode_handle,
4703558b0165SArd Biesheuvel "socionext,synquacer-pre-its",
4704558b0165SArd Biesheuvel pre_its_window,
4705558b0165SArd Biesheuvel ARRAY_SIZE(pre_its_window))) {
4706558b0165SArd Biesheuvel
4707558b0165SArd Biesheuvel its->pre_its_base = pre_its_window[0];
4708558b0165SArd Biesheuvel its->get_msi_base = its_irq_get_msi_base_pre_its;
4709558b0165SArd Biesheuvel
4710558b0165SArd Biesheuvel ids = ilog2(pre_its_window[1]) - 2;
4711576a8342SMarc Zyngier if (device_ids(its) > ids) {
4712576a8342SMarc Zyngier its->typer &= ~GITS_TYPER_DEVBITS;
4713576a8342SMarc Zyngier its->typer |= FIELD_PREP(GITS_TYPER_DEVBITS, ids - 1);
4714576a8342SMarc Zyngier }
4715558b0165SArd Biesheuvel
4716558b0165SArd Biesheuvel /* the pre-ITS breaks isolation, so disable MSI remapping */
4717dcb83f6eSJason Gunthorpe its->msi_domain_flags &= ~IRQ_DOMAIN_FLAG_ISOLATED_MSI;
4718558b0165SArd Biesheuvel return true;
4719558b0165SArd Biesheuvel }
4720558b0165SArd Biesheuvel return false;
4721558b0165SArd Biesheuvel }
4722558b0165SArd Biesheuvel
its_enable_quirk_hip07_161600802(void * data)47235c9a882eSMarc Zyngier static bool __maybe_unused its_enable_quirk_hip07_161600802(void *data)
47245c9a882eSMarc Zyngier {
47255c9a882eSMarc Zyngier struct its_node *its = data;
47265c9a882eSMarc Zyngier
47275c9a882eSMarc Zyngier /*
47285c9a882eSMarc Zyngier * Hip07 insists on using the wrong address for the VLPI
47295c9a882eSMarc Zyngier * page. Trick it into doing the right thing...
47305c9a882eSMarc Zyngier */
47315c9a882eSMarc Zyngier its->vlpi_redist_offset = SZ_128K;
47325c9a882eSMarc Zyngier return true;
4733cc2d3216SMarc Zyngier }
47344c21f3c2SMarc Zyngier
its_enable_rk3588001(void * data)4735a8707f55SSebastian Reichel static bool __maybe_unused its_enable_rk3588001(void *data)
4736a8707f55SSebastian Reichel {
4737a8707f55SSebastian Reichel struct its_node *its = data;
4738a8707f55SSebastian Reichel
4739567f67acSSebastian Reichel if (!of_machine_is_compatible("rockchip,rk3588") &&
4740567f67acSSebastian Reichel !of_machine_is_compatible("rockchip,rk3588s"))
4741a8707f55SSebastian Reichel return false;
4742a8707f55SSebastian Reichel
4743a8707f55SSebastian Reichel its->flags |= ITS_FLAGS_FORCE_NON_SHAREABLE;
4744a8707f55SSebastian Reichel gic_rdists->flags |= RDIST_FLAGS_FORCE_NON_SHAREABLE;
4745a8707f55SSebastian Reichel
4746a8707f55SSebastian Reichel return true;
4747a8707f55SSebastian Reichel }
4748a8707f55SSebastian Reichel
its_set_non_coherent(void * data)47493a0fff0fSLorenzo Pieralisi static bool its_set_non_coherent(void *data)
47503a0fff0fSLorenzo Pieralisi {
47513a0fff0fSLorenzo Pieralisi struct its_node *its = data;
47523a0fff0fSLorenzo Pieralisi
47533a0fff0fSLorenzo Pieralisi its->flags |= ITS_FLAGS_FORCE_NON_SHAREABLE;
47543a0fff0fSLorenzo Pieralisi return true;
47553a0fff0fSLorenzo Pieralisi }
47563a0fff0fSLorenzo Pieralisi
475767510ccaSRobert Richter static const struct gic_quirk its_quirks[] = {
475894100970SRobert Richter #ifdef CONFIG_CAVIUM_ERRATUM_22375
475994100970SRobert Richter {
476094100970SRobert Richter .desc = "ITS: Cavium errata 22375, 24313",
476194100970SRobert Richter .iidr = 0xa100034c, /* ThunderX pass 1.x */
476294100970SRobert Richter .mask = 0xffff0fff,
476394100970SRobert Richter .init = its_enable_quirk_cavium_22375,
476494100970SRobert Richter },
476594100970SRobert Richter #endif
4766fbf8f40eSGanapatrao Kulkarni #ifdef CONFIG_CAVIUM_ERRATUM_23144
4767fbf8f40eSGanapatrao Kulkarni {
4768fbf8f40eSGanapatrao Kulkarni .desc = "ITS: Cavium erratum 23144",
4769fbf8f40eSGanapatrao Kulkarni .iidr = 0xa100034c, /* ThunderX pass 1.x */
4770fbf8f40eSGanapatrao Kulkarni .mask = 0xffff0fff,
4771fbf8f40eSGanapatrao Kulkarni .init = its_enable_quirk_cavium_23144,
4772fbf8f40eSGanapatrao Kulkarni },
4773fbf8f40eSGanapatrao Kulkarni #endif
477490922a2dSShanker Donthineni #ifdef CONFIG_QCOM_QDF2400_ERRATUM_0065
477590922a2dSShanker Donthineni {
477690922a2dSShanker Donthineni .desc = "ITS: QDF2400 erratum 0065",
477790922a2dSShanker Donthineni .iidr = 0x00001070, /* QDF2400 ITS rev 1.x */
477890922a2dSShanker Donthineni .mask = 0xffffffff,
477990922a2dSShanker Donthineni .init = its_enable_quirk_qdf2400_e0065,
478090922a2dSShanker Donthineni },
478190922a2dSShanker Donthineni #endif
4782558b0165SArd Biesheuvel #ifdef CONFIG_SOCIONEXT_SYNQUACER_PREITS
4783558b0165SArd Biesheuvel {
4784558b0165SArd Biesheuvel /*
4785558b0165SArd Biesheuvel * The Socionext Synquacer SoC incorporates ARM's own GIC-500
4786558b0165SArd Biesheuvel * implementation, but with a 'pre-ITS' added that requires
4787558b0165SArd Biesheuvel * special handling in software.
4788558b0165SArd Biesheuvel */
4789558b0165SArd Biesheuvel .desc = "ITS: Socionext Synquacer pre-ITS",
4790558b0165SArd Biesheuvel .iidr = 0x0001143b,
4791558b0165SArd Biesheuvel .mask = 0xffffffff,
4792558b0165SArd Biesheuvel .init = its_enable_quirk_socionext_synquacer,
4793558b0165SArd Biesheuvel },
4794558b0165SArd Biesheuvel #endif
47955c9a882eSMarc Zyngier #ifdef CONFIG_HISILICON_ERRATUM_161600802
47965c9a882eSMarc Zyngier {
47975c9a882eSMarc Zyngier .desc = "ITS: Hip07 erratum 161600802",
47985c9a882eSMarc Zyngier .iidr = 0x00000004,
47995c9a882eSMarc Zyngier .mask = 0xffffffff,
48005c9a882eSMarc Zyngier .init = its_enable_quirk_hip07_161600802,
48015c9a882eSMarc Zyngier },
48025c9a882eSMarc Zyngier #endif
4803a8707f55SSebastian Reichel #ifdef CONFIG_ROCKCHIP_ERRATUM_3588001
4804a8707f55SSebastian Reichel {
4805a8707f55SSebastian Reichel .desc = "ITS: Rockchip erratum RK3588001",
4806a8707f55SSebastian Reichel .iidr = 0x0201743b,
4807a8707f55SSebastian Reichel .mask = 0xffffffff,
4808a8707f55SSebastian Reichel .init = its_enable_rk3588001,
4809a8707f55SSebastian Reichel },
4810a8707f55SSebastian Reichel #endif
481167510ccaSRobert Richter {
48123a0fff0fSLorenzo Pieralisi .desc = "ITS: non-coherent attribute",
48133a0fff0fSLorenzo Pieralisi .property = "dma-noncoherent",
48143a0fff0fSLorenzo Pieralisi .init = its_set_non_coherent,
48153a0fff0fSLorenzo Pieralisi },
48163a0fff0fSLorenzo Pieralisi {
481767510ccaSRobert Richter }
481867510ccaSRobert Richter };
481967510ccaSRobert Richter
its_enable_quirks(struct its_node * its)482067510ccaSRobert Richter static void its_enable_quirks(struct its_node *its)
482167510ccaSRobert Richter {
482267510ccaSRobert Richter u32 iidr = readl_relaxed(its->base + GITS_IIDR);
482367510ccaSRobert Richter
482467510ccaSRobert Richter gic_enable_quirks(iidr, its_quirks, its);
48253a0fff0fSLorenzo Pieralisi
48263a0fff0fSLorenzo Pieralisi if (is_of_node(its->fwnode_handle))
48273a0fff0fSLorenzo Pieralisi gic_enable_of_quirks(to_of_node(its->fwnode_handle),
48283a0fff0fSLorenzo Pieralisi its_quirks, its);
482967510ccaSRobert Richter }
483067510ccaSRobert Richter
its_save_disable(void)4831dba0bc7bSDerek Basehore static int its_save_disable(void)
4832dba0bc7bSDerek Basehore {
4833dba0bc7bSDerek Basehore struct its_node *its;
4834dba0bc7bSDerek Basehore int err = 0;
4835dba0bc7bSDerek Basehore
4836a8db7456SSebastian Andrzej Siewior raw_spin_lock(&its_lock);
4837dba0bc7bSDerek Basehore list_for_each_entry(its, &its_nodes, entry) {
4838dba0bc7bSDerek Basehore void __iomem *base;
4839dba0bc7bSDerek Basehore
4840dba0bc7bSDerek Basehore base = its->base;
4841dba0bc7bSDerek Basehore its->ctlr_save = readl_relaxed(base + GITS_CTLR);
4842dba0bc7bSDerek Basehore err = its_force_quiescent(base);
4843dba0bc7bSDerek Basehore if (err) {
4844dba0bc7bSDerek Basehore pr_err("ITS@%pa: failed to quiesce: %d\n",
4845dba0bc7bSDerek Basehore &its->phys_base, err);
4846dba0bc7bSDerek Basehore writel_relaxed(its->ctlr_save, base + GITS_CTLR);
4847dba0bc7bSDerek Basehore goto err;
4848dba0bc7bSDerek Basehore }
4849dba0bc7bSDerek Basehore
4850dba0bc7bSDerek Basehore its->cbaser_save = gits_read_cbaser(base + GITS_CBASER);
4851dba0bc7bSDerek Basehore }
4852dba0bc7bSDerek Basehore
4853dba0bc7bSDerek Basehore err:
4854dba0bc7bSDerek Basehore if (err) {
4855dba0bc7bSDerek Basehore list_for_each_entry_continue_reverse(its, &its_nodes, entry) {
4856dba0bc7bSDerek Basehore void __iomem *base;
4857dba0bc7bSDerek Basehore
4858dba0bc7bSDerek Basehore base = its->base;
4859dba0bc7bSDerek Basehore writel_relaxed(its->ctlr_save, base + GITS_CTLR);
4860dba0bc7bSDerek Basehore }
4861dba0bc7bSDerek Basehore }
4862a8db7456SSebastian Andrzej Siewior raw_spin_unlock(&its_lock);
4863dba0bc7bSDerek Basehore
4864dba0bc7bSDerek Basehore return err;
4865dba0bc7bSDerek Basehore }
4866dba0bc7bSDerek Basehore
its_restore_enable(void)4867dba0bc7bSDerek Basehore static void its_restore_enable(void)
4868dba0bc7bSDerek Basehore {
4869dba0bc7bSDerek Basehore struct its_node *its;
4870dba0bc7bSDerek Basehore int ret;
4871dba0bc7bSDerek Basehore
4872a8db7456SSebastian Andrzej Siewior raw_spin_lock(&its_lock);
4873dba0bc7bSDerek Basehore list_for_each_entry(its, &its_nodes, entry) {
4874dba0bc7bSDerek Basehore void __iomem *base;
4875dba0bc7bSDerek Basehore int i;
4876dba0bc7bSDerek Basehore
4877dba0bc7bSDerek Basehore base = its->base;
4878dba0bc7bSDerek Basehore
4879dba0bc7bSDerek Basehore /*
4880dba0bc7bSDerek Basehore * Make sure that the ITS is disabled. If it fails to quiesce,
4881dba0bc7bSDerek Basehore * don't restore it since writing to CBASER or BASER<n>
4882dba0bc7bSDerek Basehore * registers is undefined according to the GIC v3 ITS
4883dba0bc7bSDerek Basehore * Specification.
488474cde1a5SXu Qiang *
488574cde1a5SXu Qiang * Firmware resuming with the ITS enabled is terminally broken.
4886dba0bc7bSDerek Basehore */
488774cde1a5SXu Qiang WARN_ON(readl_relaxed(base + GITS_CTLR) & GITS_CTLR_ENABLE);
4888dba0bc7bSDerek Basehore ret = its_force_quiescent(base);
4889dba0bc7bSDerek Basehore if (ret) {
4890dba0bc7bSDerek Basehore pr_err("ITS@%pa: failed to quiesce on resume: %d\n",
4891dba0bc7bSDerek Basehore &its->phys_base, ret);
4892dba0bc7bSDerek Basehore continue;
4893dba0bc7bSDerek Basehore }
4894dba0bc7bSDerek Basehore
4895dba0bc7bSDerek Basehore gits_write_cbaser(its->cbaser_save, base + GITS_CBASER);
4896dba0bc7bSDerek Basehore
4897dba0bc7bSDerek Basehore /*
4898dba0bc7bSDerek Basehore * Writing CBASER resets CREADR to 0, so make CWRITER and
4899dba0bc7bSDerek Basehore * cmd_write line up with it.
4900dba0bc7bSDerek Basehore */
4901dba0bc7bSDerek Basehore its->cmd_write = its->cmd_base;
4902dba0bc7bSDerek Basehore gits_write_cwriter(0, base + GITS_CWRITER);
4903dba0bc7bSDerek Basehore
4904dba0bc7bSDerek Basehore /* Restore GITS_BASER from the value cache. */
4905dba0bc7bSDerek Basehore for (i = 0; i < GITS_BASER_NR_REGS; i++) {
4906dba0bc7bSDerek Basehore struct its_baser *baser = &its->tables[i];
4907dba0bc7bSDerek Basehore
4908dba0bc7bSDerek Basehore if (!(baser->val & GITS_BASER_VALID))
4909dba0bc7bSDerek Basehore continue;
4910dba0bc7bSDerek Basehore
4911dba0bc7bSDerek Basehore its_write_baser(its, baser, baser->val);
4912dba0bc7bSDerek Basehore }
4913dba0bc7bSDerek Basehore writel_relaxed(its->ctlr_save, base + GITS_CTLR);
4914920181ceSDerek Basehore
4915920181ceSDerek Basehore /*
4916920181ceSDerek Basehore * Reinit the collection if it's stored in the ITS. This is
4917920181ceSDerek Basehore * indicated by the col_id being less than the HCC field.
4918920181ceSDerek Basehore * CID < HCC as specified in the GIC v3 Documentation.
4919920181ceSDerek Basehore */
4920920181ceSDerek Basehore if (its->collections[smp_processor_id()].col_id <
4921920181ceSDerek Basehore GITS_TYPER_HCC(gic_read_typer(base + GITS_TYPER)))
4922920181ceSDerek Basehore its_cpu_init_collection(its);
4923dba0bc7bSDerek Basehore }
4924a8db7456SSebastian Andrzej Siewior raw_spin_unlock(&its_lock);
4925dba0bc7bSDerek Basehore }
4926dba0bc7bSDerek Basehore
4927dba0bc7bSDerek Basehore static struct syscore_ops its_syscore_ops = {
4928dba0bc7bSDerek Basehore .suspend = its_save_disable,
4929dba0bc7bSDerek Basehore .resume = its_restore_enable,
4930dba0bc7bSDerek Basehore };
4931dba0bc7bSDerek Basehore
its_map_one(struct resource * res,int * err)4932c733ebb7SMarc Zyngier static void __init __iomem *its_map_one(struct resource *res, int *err)
4933c733ebb7SMarc Zyngier {
4934c733ebb7SMarc Zyngier void __iomem *its_base;
4935c733ebb7SMarc Zyngier u32 val;
4936c733ebb7SMarc Zyngier
4937c733ebb7SMarc Zyngier its_base = ioremap(res->start, SZ_64K);
4938c733ebb7SMarc Zyngier if (!its_base) {
4939c733ebb7SMarc Zyngier pr_warn("ITS@%pa: Unable to map ITS registers\n", &res->start);
4940c733ebb7SMarc Zyngier *err = -ENOMEM;
4941c733ebb7SMarc Zyngier return NULL;
4942c733ebb7SMarc Zyngier }
4943c733ebb7SMarc Zyngier
4944c733ebb7SMarc Zyngier val = readl_relaxed(its_base + GITS_PIDR2) & GIC_PIDR2_ARCH_MASK;
4945c733ebb7SMarc Zyngier if (val != 0x30 && val != 0x40) {
4946c733ebb7SMarc Zyngier pr_warn("ITS@%pa: No ITS detected, giving up\n", &res->start);
4947c733ebb7SMarc Zyngier *err = -ENODEV;
4948c733ebb7SMarc Zyngier goto out_unmap;
4949c733ebb7SMarc Zyngier }
4950c733ebb7SMarc Zyngier
4951c733ebb7SMarc Zyngier *err = its_force_quiescent(its_base);
4952c733ebb7SMarc Zyngier if (*err) {
4953c733ebb7SMarc Zyngier pr_warn("ITS@%pa: Failed to quiesce, giving up\n", &res->start);
4954c733ebb7SMarc Zyngier goto out_unmap;
4955c733ebb7SMarc Zyngier }
4956c733ebb7SMarc Zyngier
4957c733ebb7SMarc Zyngier return its_base;
4958c733ebb7SMarc Zyngier
4959c733ebb7SMarc Zyngier out_unmap:
4960c733ebb7SMarc Zyngier iounmap(its_base);
4961c733ebb7SMarc Zyngier return NULL;
4962c733ebb7SMarc Zyngier }
4963c733ebb7SMarc Zyngier
its_init_domain(struct its_node * its)49649585a495SMarc Zyngier static int its_init_domain(struct its_node *its)
4965d14ae5e6STomasz Nowicki {
4966d14ae5e6STomasz Nowicki struct irq_domain *inner_domain;
4967d14ae5e6STomasz Nowicki struct msi_domain_info *info;
4968d14ae5e6STomasz Nowicki
4969d14ae5e6STomasz Nowicki info = kzalloc(sizeof(*info), GFP_KERNEL);
4970d14ae5e6STomasz Nowicki if (!info)
4971d14ae5e6STomasz Nowicki return -ENOMEM;
4972d14ae5e6STomasz Nowicki
49731e46e040SJohan Hovold info->ops = &its_msi_domain_ops;
49741e46e040SJohan Hovold info->data = its;
49751e46e040SJohan Hovold
49761e46e040SJohan Hovold inner_domain = irq_domain_create_hierarchy(its_parent,
49771e46e040SJohan Hovold its->msi_domain_flags, 0,
49789585a495SMarc Zyngier its->fwnode_handle, &its_domain_ops,
49791e46e040SJohan Hovold info);
4980d14ae5e6STomasz Nowicki if (!inner_domain) {
4981d14ae5e6STomasz Nowicki kfree(info);
4982d14ae5e6STomasz Nowicki return -ENOMEM;
4983d14ae5e6STomasz Nowicki }
4984d14ae5e6STomasz Nowicki
498596f0d93aSMarc Zyngier irq_domain_update_bus_token(inner_domain, DOMAIN_BUS_NEXUS);
4986d14ae5e6STomasz Nowicki
4987d14ae5e6STomasz Nowicki return 0;
4988d14ae5e6STomasz Nowicki }
4989d14ae5e6STomasz Nowicki
its_init_vpe_domain(void)49908fff27aeSMarc Zyngier static int its_init_vpe_domain(void)
49918fff27aeSMarc Zyngier {
499220b3d54eSMarc Zyngier struct its_node *its;
499320b3d54eSMarc Zyngier u32 devid;
499420b3d54eSMarc Zyngier int entries;
499520b3d54eSMarc Zyngier
499620b3d54eSMarc Zyngier if (gic_rdists->has_direct_lpi) {
499720b3d54eSMarc Zyngier pr_info("ITS: Using DirectLPI for VPE invalidation\n");
499820b3d54eSMarc Zyngier return 0;
499920b3d54eSMarc Zyngier }
500020b3d54eSMarc Zyngier
500120b3d54eSMarc Zyngier /* Any ITS will do, even if not v4 */
500220b3d54eSMarc Zyngier its = list_first_entry(&its_nodes, struct its_node, entry);
500320b3d54eSMarc Zyngier
500420b3d54eSMarc Zyngier entries = roundup_pow_of_two(nr_cpu_ids);
50056396bb22SKees Cook vpe_proxy.vpes = kcalloc(entries, sizeof(*vpe_proxy.vpes),
500620b3d54eSMarc Zyngier GFP_KERNEL);
5007944a1a17SZhen Lei if (!vpe_proxy.vpes)
500820b3d54eSMarc Zyngier return -ENOMEM;
500920b3d54eSMarc Zyngier
501020b3d54eSMarc Zyngier /* Use the last possible DevID */
5011576a8342SMarc Zyngier devid = GENMASK(device_ids(its) - 1, 0);
501220b3d54eSMarc Zyngier vpe_proxy.dev = its_create_device(its, devid, entries, false);
501320b3d54eSMarc Zyngier if (!vpe_proxy.dev) {
501420b3d54eSMarc Zyngier kfree(vpe_proxy.vpes);
501520b3d54eSMarc Zyngier pr_err("ITS: Can't allocate GICv4 proxy device\n");
501620b3d54eSMarc Zyngier return -ENOMEM;
501720b3d54eSMarc Zyngier }
501820b3d54eSMarc Zyngier
5019c427a475SShanker Donthineni BUG_ON(entries > vpe_proxy.dev->nr_ites);
502020b3d54eSMarc Zyngier
502120b3d54eSMarc Zyngier raw_spin_lock_init(&vpe_proxy.lock);
502220b3d54eSMarc Zyngier vpe_proxy.next_victim = 0;
502320b3d54eSMarc Zyngier pr_info("ITS: Allocated DevID %x as GICv4 proxy device (%d slots)\n",
502420b3d54eSMarc Zyngier devid, vpe_proxy.dev->nr_ites);
502520b3d54eSMarc Zyngier
50268fff27aeSMarc Zyngier return 0;
50278fff27aeSMarc Zyngier }
50288fff27aeSMarc Zyngier
its_compute_its_list_map(struct its_node * its)50299585a495SMarc Zyngier static int __init its_compute_its_list_map(struct its_node *its)
50303dfa576bSMarc Zyngier {
50313dfa576bSMarc Zyngier int its_number;
50323dfa576bSMarc Zyngier u32 ctlr;
50333dfa576bSMarc Zyngier
50343dfa576bSMarc Zyngier /*
50353dfa576bSMarc Zyngier * This is assumed to be done early enough that we're
50363dfa576bSMarc Zyngier * guaranteed to be single-threaded, hence no
50373dfa576bSMarc Zyngier * locking. Should this change, we should address
50383dfa576bSMarc Zyngier * this.
50393dfa576bSMarc Zyngier */
5040ab60491eSMarc Zyngier its_number = find_first_zero_bit(&its_list_map, GICv4_ITS_LIST_MAX);
5041ab60491eSMarc Zyngier if (its_number >= GICv4_ITS_LIST_MAX) {
50423dfa576bSMarc Zyngier pr_err("ITS@%pa: No ITSList entry available!\n",
50439585a495SMarc Zyngier &its->phys_base);
50443dfa576bSMarc Zyngier return -EINVAL;
50453dfa576bSMarc Zyngier }
50463dfa576bSMarc Zyngier
50479585a495SMarc Zyngier ctlr = readl_relaxed(its->base + GITS_CTLR);
50483dfa576bSMarc Zyngier ctlr &= ~GITS_CTLR_ITS_NUMBER;
50493dfa576bSMarc Zyngier ctlr |= its_number << GITS_CTLR_ITS_NUMBER_SHIFT;
50509585a495SMarc Zyngier writel_relaxed(ctlr, its->base + GITS_CTLR);
50519585a495SMarc Zyngier ctlr = readl_relaxed(its->base + GITS_CTLR);
50523dfa576bSMarc Zyngier if ((ctlr & GITS_CTLR_ITS_NUMBER) != (its_number << GITS_CTLR_ITS_NUMBER_SHIFT)) {
50533dfa576bSMarc Zyngier its_number = ctlr & GITS_CTLR_ITS_NUMBER;
50543dfa576bSMarc Zyngier its_number >>= GITS_CTLR_ITS_NUMBER_SHIFT;
50553dfa576bSMarc Zyngier }
50563dfa576bSMarc Zyngier
50573dfa576bSMarc Zyngier if (test_and_set_bit(its_number, &its_list_map)) {
50583dfa576bSMarc Zyngier pr_err("ITS@%pa: Duplicate ITSList entry %d\n",
50599585a495SMarc Zyngier &its->phys_base, its_number);
50603dfa576bSMarc Zyngier return -EINVAL;
50613dfa576bSMarc Zyngier }
50623dfa576bSMarc Zyngier
50633dfa576bSMarc Zyngier return its_number;
50643dfa576bSMarc Zyngier }
50653dfa576bSMarc Zyngier
its_probe_one(struct its_node * its)50669585a495SMarc Zyngier static int __init its_probe_one(struct its_node *its)
50674c21f3c2SMarc Zyngier {
50689585a495SMarc Zyngier u64 baser, tmp;
5069539d3782SShanker Donthineni struct page *page;
5070c733ebb7SMarc Zyngier u32 ctlr;
50714c21f3c2SMarc Zyngier int err;
50724c21f3c2SMarc Zyngier
507391a80fffSMarc Zyngier its_enable_quirks(its);
507491a80fffSMarc Zyngier
50750dd57fedSMarc Zyngier if (is_v4(its)) {
50769585a495SMarc Zyngier if (!(its->typer & GITS_TYPER_VMOVP)) {
50779585a495SMarc Zyngier err = its_compute_its_list_map(its);
50783dfa576bSMarc Zyngier if (err < 0)
50799585a495SMarc Zyngier goto out;
50803dfa576bSMarc Zyngier
5081debf6d02SMarc Zyngier its->list_nr = err;
5082debf6d02SMarc Zyngier
50833dfa576bSMarc Zyngier pr_info("ITS@%pa: Using ITS number %d\n",
50849585a495SMarc Zyngier &its->phys_base, err);
50853dfa576bSMarc Zyngier } else {
50869585a495SMarc Zyngier pr_info("ITS@%pa: Single VMOVP capable\n", &its->phys_base);
50873dfa576bSMarc Zyngier }
50885e516846SMarc Zyngier
50895e516846SMarc Zyngier if (is_v4_1(its)) {
50909585a495SMarc Zyngier u32 svpet = FIELD_GET(GITS_TYPER_SVPET, its->typer);
50915e46a484SMarc Zyngier
50929585a495SMarc Zyngier its->sgir_base = ioremap(its->phys_base + SZ_128K, SZ_64K);
50935e46a484SMarc Zyngier if (!its->sgir_base) {
50945e46a484SMarc Zyngier err = -ENOMEM;
50959585a495SMarc Zyngier goto out;
50965e46a484SMarc Zyngier }
50975e46a484SMarc Zyngier
50989585a495SMarc Zyngier its->mpidr = readl_relaxed(its->base + GITS_MPIDR);
50995e516846SMarc Zyngier
51005e516846SMarc Zyngier pr_info("ITS@%pa: Using GICv4.1 mode %08x %08x\n",
51019585a495SMarc Zyngier &its->phys_base, its->mpidr, svpet);
51025e516846SMarc Zyngier }
51033dfa576bSMarc Zyngier }
51043dfa576bSMarc Zyngier
5105539d3782SShanker Donthineni page = alloc_pages_node(its->numa_node, GFP_KERNEL | __GFP_ZERO,
51065bc13c2cSRobert Richter get_order(ITS_CMD_QUEUE_SZ));
5107539d3782SShanker Donthineni if (!page) {
51084c21f3c2SMarc Zyngier err = -ENOMEM;
51095e46a484SMarc Zyngier goto out_unmap_sgir;
51104c21f3c2SMarc Zyngier }
5111539d3782SShanker Donthineni its->cmd_base = (void *)page_address(page);
51124c21f3c2SMarc Zyngier its->cmd_write = its->cmd_base;
51134c21f3c2SMarc Zyngier
51140e0b0f69SShanker Donthineni err = its_alloc_tables(its);
51154c21f3c2SMarc Zyngier if (err)
51164c21f3c2SMarc Zyngier goto out_free_cmd;
51174c21f3c2SMarc Zyngier
51184c21f3c2SMarc Zyngier err = its_alloc_collections(its);
51194c21f3c2SMarc Zyngier if (err)
51204c21f3c2SMarc Zyngier goto out_free_tables;
51214c21f3c2SMarc Zyngier
51224c21f3c2SMarc Zyngier baser = (virt_to_phys(its->cmd_base) |
51232fd632a0SShanker Donthineni GITS_CBASER_RaWaWb |
51244c21f3c2SMarc Zyngier GITS_CBASER_InnerShareable |
51254c21f3c2SMarc Zyngier (ITS_CMD_QUEUE_SZ / SZ_4K - 1) |
51264c21f3c2SMarc Zyngier GITS_CBASER_VALID);
51274c21f3c2SMarc Zyngier
51280968a619SVladimir Murzin gits_write_cbaser(baser, its->base + GITS_CBASER);
51290968a619SVladimir Murzin tmp = gits_read_cbaser(its->base + GITS_CBASER);
51304c21f3c2SMarc Zyngier
5131a8707f55SSebastian Reichel if (its->flags & ITS_FLAGS_FORCE_NON_SHAREABLE)
5132a8707f55SSebastian Reichel tmp &= ~GITS_CBASER_SHAREABILITY_MASK;
5133a8707f55SSebastian Reichel
51344ad3e363SMarc Zyngier if ((tmp ^ baser) & GITS_CBASER_SHAREABILITY_MASK) {
5135241a386cSMarc Zyngier if (!(tmp & GITS_CBASER_SHAREABILITY_MASK)) {
5136241a386cSMarc Zyngier /*
5137241a386cSMarc Zyngier * The HW reports non-shareable, we must
5138241a386cSMarc Zyngier * remove the cacheability attributes as
5139241a386cSMarc Zyngier * well.
5140241a386cSMarc Zyngier */
5141241a386cSMarc Zyngier baser &= ~(GITS_CBASER_SHAREABILITY_MASK |
5142241a386cSMarc Zyngier GITS_CBASER_CACHEABILITY_MASK);
5143241a386cSMarc Zyngier baser |= GITS_CBASER_nC;
51440968a619SVladimir Murzin gits_write_cbaser(baser, its->base + GITS_CBASER);
5145241a386cSMarc Zyngier }
51464c21f3c2SMarc Zyngier pr_info("ITS: using cache flushing for cmd queue\n");
51474c21f3c2SMarc Zyngier its->flags |= ITS_FLAGS_CMDQ_NEEDS_FLUSHING;
51484c21f3c2SMarc Zyngier }
51494c21f3c2SMarc Zyngier
51500968a619SVladimir Murzin gits_write_cwriter(0, its->base + GITS_CWRITER);
51513dfa576bSMarc Zyngier ctlr = readl_relaxed(its->base + GITS_CTLR);
5152d51c4b4dSMarc Zyngier ctlr |= GITS_CTLR_ENABLE;
51530dd57fedSMarc Zyngier if (is_v4(its))
5154d51c4b4dSMarc Zyngier ctlr |= GITS_CTLR_ImDe;
5155d51c4b4dSMarc Zyngier writel_relaxed(ctlr, its->base + GITS_CTLR);
5156241a386cSMarc Zyngier
51579585a495SMarc Zyngier err = its_init_domain(its);
5158d14ae5e6STomasz Nowicki if (err)
515954456db9SMarc Zyngier goto out_free_tables;
51604c21f3c2SMarc Zyngier
5161a8db7456SSebastian Andrzej Siewior raw_spin_lock(&its_lock);
51624c21f3c2SMarc Zyngier list_add(&its->entry, &its_nodes);
5163a8db7456SSebastian Andrzej Siewior raw_spin_unlock(&its_lock);
51644c21f3c2SMarc Zyngier
51654c21f3c2SMarc Zyngier return 0;
51664c21f3c2SMarc Zyngier
51674c21f3c2SMarc Zyngier out_free_tables:
51684c21f3c2SMarc Zyngier its_free_tables(its);
51694c21f3c2SMarc Zyngier out_free_cmd:
51705bc13c2cSRobert Richter free_pages((unsigned long)its->cmd_base, get_order(ITS_CMD_QUEUE_SZ));
51715e46a484SMarc Zyngier out_unmap_sgir:
51725e46a484SMarc Zyngier if (its->sgir_base)
51735e46a484SMarc Zyngier iounmap(its->sgir_base);
51749585a495SMarc Zyngier out:
51759585a495SMarc Zyngier pr_err("ITS@%pa: failed probing (%d)\n", &its->phys_base, err);
51764c21f3c2SMarc Zyngier return err;
51774c21f3c2SMarc Zyngier }
51784c21f3c2SMarc Zyngier
gic_rdists_supports_plpis(void)51794c21f3c2SMarc Zyngier static bool gic_rdists_supports_plpis(void)
51804c21f3c2SMarc Zyngier {
5181589ce5f4SMarc Zyngier return !!(gic_read_typer(gic_data_rdist_rd_base() + GICR_TYPER) & GICR_TYPER_PLPIS);
51824c21f3c2SMarc Zyngier }
51834c21f3c2SMarc Zyngier
redist_disable_lpis(void)51846eb486b6SShanker Donthineni static int redist_disable_lpis(void)
51854c21f3c2SMarc Zyngier {
51866eb486b6SShanker Donthineni void __iomem *rbase = gic_data_rdist_rd_base();
51876eb486b6SShanker Donthineni u64 timeout = USEC_PER_SEC;
51886eb486b6SShanker Donthineni u64 val;
51896eb486b6SShanker Donthineni
51904c21f3c2SMarc Zyngier if (!gic_rdists_supports_plpis()) {
51914c21f3c2SMarc Zyngier pr_info("CPU%d: LPIs not supported\n", smp_processor_id());
51924c21f3c2SMarc Zyngier return -ENXIO;
51934c21f3c2SMarc Zyngier }
51946eb486b6SShanker Donthineni
51956eb486b6SShanker Donthineni val = readl_relaxed(rbase + GICR_CTLR);
51966eb486b6SShanker Donthineni if (!(val & GICR_CTLR_ENABLE_LPIS))
51976eb486b6SShanker Donthineni return 0;
51986eb486b6SShanker Donthineni
519911e37d35SMarc Zyngier /*
520011e37d35SMarc Zyngier * If coming via a CPU hotplug event, we don't need to disable
520111e37d35SMarc Zyngier * LPIs before trying to re-enable them. They are already
520211e37d35SMarc Zyngier * configured and all is well in the world.
5203c440a9d9SMarc Zyngier *
5204c440a9d9SMarc Zyngier * If running with preallocated tables, there is nothing to do.
520511e37d35SMarc Zyngier */
5206c0cdc890SValentin Schneider if ((gic_data_rdist()->flags & RD_LOCAL_LPI_ENABLED) ||
5207c440a9d9SMarc Zyngier (gic_rdists->flags & RDIST_FLAGS_RD_TABLES_PREALLOCATED))
520811e37d35SMarc Zyngier return 0;
520911e37d35SMarc Zyngier
521011e37d35SMarc Zyngier /*
521111e37d35SMarc Zyngier * From that point on, we only try to do some damage control.
521211e37d35SMarc Zyngier */
521311e37d35SMarc Zyngier pr_warn("GICv3: CPU%d: Booted with LPIs enabled, memory probably corrupted\n",
52146eb486b6SShanker Donthineni smp_processor_id());
52156eb486b6SShanker Donthineni add_taint(TAINT_CRAP, LOCKDEP_STILL_OK);
52166eb486b6SShanker Donthineni
52176eb486b6SShanker Donthineni /* Disable LPIs */
52186eb486b6SShanker Donthineni val &= ~GICR_CTLR_ENABLE_LPIS;
52196eb486b6SShanker Donthineni writel_relaxed(val, rbase + GICR_CTLR);
52206eb486b6SShanker Donthineni
52216eb486b6SShanker Donthineni /* Make sure any change to GICR_CTLR is observable by the GIC */
52226eb486b6SShanker Donthineni dsb(sy);
52236eb486b6SShanker Donthineni
52246eb486b6SShanker Donthineni /*
52256eb486b6SShanker Donthineni * Software must observe RWP==0 after clearing GICR_CTLR.EnableLPIs
52266eb486b6SShanker Donthineni * from 1 to 0 before programming GICR_PEND{PROP}BASER registers.
52276eb486b6SShanker Donthineni * Error out if we time out waiting for RWP to clear.
52286eb486b6SShanker Donthineni */
52296eb486b6SShanker Donthineni while (readl_relaxed(rbase + GICR_CTLR) & GICR_CTLR_RWP) {
52306eb486b6SShanker Donthineni if (!timeout) {
52316eb486b6SShanker Donthineni pr_err("CPU%d: Timeout while disabling LPIs\n",
52326eb486b6SShanker Donthineni smp_processor_id());
52336eb486b6SShanker Donthineni return -ETIMEDOUT;
52346eb486b6SShanker Donthineni }
52356eb486b6SShanker Donthineni udelay(1);
52366eb486b6SShanker Donthineni timeout--;
52376eb486b6SShanker Donthineni }
52386eb486b6SShanker Donthineni
52396eb486b6SShanker Donthineni /*
52406eb486b6SShanker Donthineni * After it has been written to 1, it is IMPLEMENTATION
52416eb486b6SShanker Donthineni * DEFINED whether GICR_CTLR.EnableLPI becomes RES1 or can be
52426eb486b6SShanker Donthineni * cleared to 0. Error out if clearing the bit failed.
52436eb486b6SShanker Donthineni */
52446eb486b6SShanker Donthineni if (readl_relaxed(rbase + GICR_CTLR) & GICR_CTLR_ENABLE_LPIS) {
52456eb486b6SShanker Donthineni pr_err("CPU%d: Failed to disable LPIs\n", smp_processor_id());
52466eb486b6SShanker Donthineni return -EBUSY;
52476eb486b6SShanker Donthineni }
52486eb486b6SShanker Donthineni
52496eb486b6SShanker Donthineni return 0;
52506eb486b6SShanker Donthineni }
52516eb486b6SShanker Donthineni
its_cpu_init(void)52526eb486b6SShanker Donthineni int its_cpu_init(void)
52536eb486b6SShanker Donthineni {
52546eb486b6SShanker Donthineni if (!list_empty(&its_nodes)) {
52556eb486b6SShanker Donthineni int ret;
52566eb486b6SShanker Donthineni
52576eb486b6SShanker Donthineni ret = redist_disable_lpis();
52586eb486b6SShanker Donthineni if (ret)
52596eb486b6SShanker Donthineni return ret;
52606eb486b6SShanker Donthineni
52614c21f3c2SMarc Zyngier its_cpu_init_lpis();
5262920181ceSDerek Basehore its_cpu_init_collections();
52634c21f3c2SMarc Zyngier }
52644c21f3c2SMarc Zyngier
52654c21f3c2SMarc Zyngier return 0;
52664c21f3c2SMarc Zyngier }
52674c21f3c2SMarc Zyngier
rdist_memreserve_cpuhp_cleanup_workfn(struct work_struct * work)5268835f442fSValentin Schneider static void rdist_memreserve_cpuhp_cleanup_workfn(struct work_struct *work)
5269835f442fSValentin Schneider {
5270835f442fSValentin Schneider cpuhp_remove_state_nocalls(gic_rdists->cpuhp_memreserve_state);
5271835f442fSValentin Schneider gic_rdists->cpuhp_memreserve_state = CPUHP_INVALID;
5272835f442fSValentin Schneider }
5273835f442fSValentin Schneider
5274835f442fSValentin Schneider static DECLARE_WORK(rdist_memreserve_cpuhp_cleanup_work,
5275835f442fSValentin Schneider rdist_memreserve_cpuhp_cleanup_workfn);
5276835f442fSValentin Schneider
its_cpu_memreserve_lpi(unsigned int cpu)5277d23bc2bcSValentin Schneider static int its_cpu_memreserve_lpi(unsigned int cpu)
5278d23bc2bcSValentin Schneider {
5279d23bc2bcSValentin Schneider struct page *pend_page;
5280d23bc2bcSValentin Schneider int ret = 0;
5281d23bc2bcSValentin Schneider
5282d23bc2bcSValentin Schneider /* This gets to run exactly once per CPU */
5283d23bc2bcSValentin Schneider if (gic_data_rdist()->flags & RD_LOCAL_MEMRESERVE_DONE)
5284d23bc2bcSValentin Schneider return 0;
5285d23bc2bcSValentin Schneider
5286d23bc2bcSValentin Schneider pend_page = gic_data_rdist()->pend_page;
5287d23bc2bcSValentin Schneider if (WARN_ON(!pend_page)) {
5288d23bc2bcSValentin Schneider ret = -ENOMEM;
5289d23bc2bcSValentin Schneider goto out;
5290d23bc2bcSValentin Schneider }
5291d23bc2bcSValentin Schneider /*
5292d23bc2bcSValentin Schneider * If the pending table was pre-programmed, free the memory we
5293d23bc2bcSValentin Schneider * preemptively allocated. Otherwise, reserve that memory for
5294d23bc2bcSValentin Schneider * later kexecs.
5295d23bc2bcSValentin Schneider */
5296d23bc2bcSValentin Schneider if (gic_data_rdist()->flags & RD_LOCAL_PENDTABLE_PREALLOCATED) {
5297d23bc2bcSValentin Schneider its_free_pending_table(pend_page);
5298d23bc2bcSValentin Schneider gic_data_rdist()->pend_page = NULL;
5299d23bc2bcSValentin Schneider } else {
5300d23bc2bcSValentin Schneider phys_addr_t paddr = page_to_phys(pend_page);
5301d23bc2bcSValentin Schneider WARN_ON(gic_reserve_range(paddr, LPI_PENDBASE_SZ));
5302d23bc2bcSValentin Schneider }
5303d23bc2bcSValentin Schneider
5304d23bc2bcSValentin Schneider out:
5305835f442fSValentin Schneider /* Last CPU being brought up gets to issue the cleanup */
530616436f70SArd Biesheuvel if (!IS_ENABLED(CONFIG_SMP) ||
530716436f70SArd Biesheuvel cpumask_equal(&cpus_booted_once_mask, cpu_possible_mask))
5308835f442fSValentin Schneider schedule_work(&rdist_memreserve_cpuhp_cleanup_work);
5309835f442fSValentin Schneider
5310d23bc2bcSValentin Schneider gic_data_rdist()->flags |= RD_LOCAL_MEMRESERVE_DONE;
5311d23bc2bcSValentin Schneider return ret;
5312d23bc2bcSValentin Schneider }
5313d23bc2bcSValentin Schneider
5314c733ebb7SMarc Zyngier /* Mark all the BASER registers as invalid before they get reprogrammed */
its_reset_one(struct resource * res)5315c733ebb7SMarc Zyngier static int __init its_reset_one(struct resource *res)
5316c733ebb7SMarc Zyngier {
5317c733ebb7SMarc Zyngier void __iomem *its_base;
5318c733ebb7SMarc Zyngier int err, i;
5319c733ebb7SMarc Zyngier
5320c733ebb7SMarc Zyngier its_base = its_map_one(res, &err);
5321c733ebb7SMarc Zyngier if (!its_base)
5322c733ebb7SMarc Zyngier return err;
5323c733ebb7SMarc Zyngier
5324c733ebb7SMarc Zyngier for (i = 0; i < GITS_BASER_NR_REGS; i++)
5325c733ebb7SMarc Zyngier gits_write_baser(0, its_base + GITS_BASER + (i << 3));
5326c733ebb7SMarc Zyngier
5327c733ebb7SMarc Zyngier iounmap(its_base);
5328c733ebb7SMarc Zyngier return 0;
5329c733ebb7SMarc Zyngier }
5330c733ebb7SMarc Zyngier
5331935bba7cSArvind Yadav static const struct of_device_id its_device_id[] = {
53324c21f3c2SMarc Zyngier { .compatible = "arm,gic-v3-its", },
53334c21f3c2SMarc Zyngier {},
53344c21f3c2SMarc Zyngier };
53354c21f3c2SMarc Zyngier
its_node_init(struct resource * res,struct fwnode_handle * handle,int numa_node)53369585a495SMarc Zyngier static struct its_node __init *its_node_init(struct resource *res,
53379585a495SMarc Zyngier struct fwnode_handle *handle, int numa_node)
53389585a495SMarc Zyngier {
53399585a495SMarc Zyngier void __iomem *its_base;
53409585a495SMarc Zyngier struct its_node *its;
53419585a495SMarc Zyngier int err;
53429585a495SMarc Zyngier
53439585a495SMarc Zyngier its_base = its_map_one(res, &err);
53449585a495SMarc Zyngier if (!its_base)
53459585a495SMarc Zyngier return NULL;
53469585a495SMarc Zyngier
53479585a495SMarc Zyngier pr_info("ITS %pR\n", res);
53489585a495SMarc Zyngier
53499585a495SMarc Zyngier its = kzalloc(sizeof(*its), GFP_KERNEL);
53509585a495SMarc Zyngier if (!its)
53519585a495SMarc Zyngier goto out_unmap;
53529585a495SMarc Zyngier
53539585a495SMarc Zyngier raw_spin_lock_init(&its->lock);
53549585a495SMarc Zyngier mutex_init(&its->dev_alloc_lock);
53559585a495SMarc Zyngier INIT_LIST_HEAD(&its->entry);
53569585a495SMarc Zyngier INIT_LIST_HEAD(&its->its_device_list);
53579585a495SMarc Zyngier
53589585a495SMarc Zyngier its->typer = gic_read_typer(its_base + GITS_TYPER);
53599585a495SMarc Zyngier its->base = its_base;
53609585a495SMarc Zyngier its->phys_base = res->start;
5361f199bf5bSMarc Zyngier its->get_msi_base = its_irq_get_msi_base;
5362f199bf5bSMarc Zyngier its->msi_domain_flags = IRQ_DOMAIN_FLAG_ISOLATED_MSI;
53639585a495SMarc Zyngier
53649585a495SMarc Zyngier its->numa_node = numa_node;
53659585a495SMarc Zyngier its->fwnode_handle = handle;
53669585a495SMarc Zyngier
53679585a495SMarc Zyngier return its;
53689585a495SMarc Zyngier
53699585a495SMarc Zyngier out_unmap:
53709585a495SMarc Zyngier iounmap(its_base);
53719585a495SMarc Zyngier return NULL;
53729585a495SMarc Zyngier }
53739585a495SMarc Zyngier
its_node_destroy(struct its_node * its)53749585a495SMarc Zyngier static void its_node_destroy(struct its_node *its)
53759585a495SMarc Zyngier {
53769585a495SMarc Zyngier iounmap(its->base);
53779585a495SMarc Zyngier kfree(its);
53789585a495SMarc Zyngier }
53799585a495SMarc Zyngier
its_of_probe(struct device_node * node)5380db40f0a7STomasz Nowicki static int __init its_of_probe(struct device_node *node)
53814c21f3c2SMarc Zyngier {
53824c21f3c2SMarc Zyngier struct device_node *np;
5383db40f0a7STomasz Nowicki struct resource res;
53849585a495SMarc Zyngier int err;
53854c21f3c2SMarc Zyngier
5386c733ebb7SMarc Zyngier /*
5387c733ebb7SMarc Zyngier * Make sure *all* the ITS are reset before we probe any, as
5388c733ebb7SMarc Zyngier * they may be sharing memory. If any of the ITS fails to
5389c733ebb7SMarc Zyngier * reset, don't even try to go any further, as this could
5390c733ebb7SMarc Zyngier * result in something even worse.
5391c733ebb7SMarc Zyngier */
5392c733ebb7SMarc Zyngier for (np = of_find_matching_node(node, its_device_id); np;
5393c733ebb7SMarc Zyngier np = of_find_matching_node(np, its_device_id)) {
5394c733ebb7SMarc Zyngier if (!of_device_is_available(np) ||
5395c733ebb7SMarc Zyngier !of_property_read_bool(np, "msi-controller") ||
5396c733ebb7SMarc Zyngier of_address_to_resource(np, 0, &res))
5397c733ebb7SMarc Zyngier continue;
5398c733ebb7SMarc Zyngier
5399c733ebb7SMarc Zyngier err = its_reset_one(&res);
5400c733ebb7SMarc Zyngier if (err)
5401c733ebb7SMarc Zyngier return err;
5402c733ebb7SMarc Zyngier }
5403c733ebb7SMarc Zyngier
54044c21f3c2SMarc Zyngier for (np = of_find_matching_node(node, its_device_id); np;
54054c21f3c2SMarc Zyngier np = of_find_matching_node(np, its_device_id)) {
54069585a495SMarc Zyngier struct its_node *its;
54079585a495SMarc Zyngier
540895a25625SStephen Boyd if (!of_device_is_available(np))
540995a25625SStephen Boyd continue;
5410d14ae5e6STomasz Nowicki if (!of_property_read_bool(np, "msi-controller")) {
5411e81f54c6SRob Herring pr_warn("%pOF: no msi-controller property, ITS ignored\n",
5412e81f54c6SRob Herring np);
5413d14ae5e6STomasz Nowicki continue;
5414d14ae5e6STomasz Nowicki }
5415d14ae5e6STomasz Nowicki
5416db40f0a7STomasz Nowicki if (of_address_to_resource(np, 0, &res)) {
5417e81f54c6SRob Herring pr_warn("%pOF: no regs?\n", np);
5418db40f0a7STomasz Nowicki continue;
54194c21f3c2SMarc Zyngier }
54204c21f3c2SMarc Zyngier
54219585a495SMarc Zyngier
54229585a495SMarc Zyngier its = its_node_init(&res, &np->fwnode, of_node_to_nid(np));
54239585a495SMarc Zyngier if (!its)
54249585a495SMarc Zyngier return -ENOMEM;
54259585a495SMarc Zyngier
54269585a495SMarc Zyngier err = its_probe_one(its);
54279585a495SMarc Zyngier if (err) {
54289585a495SMarc Zyngier its_node_destroy(its);
54299585a495SMarc Zyngier return err;
54309585a495SMarc Zyngier }
5431db40f0a7STomasz Nowicki }
5432db40f0a7STomasz Nowicki return 0;
5433db40f0a7STomasz Nowicki }
5434db40f0a7STomasz Nowicki
54353f010cf1STomasz Nowicki #ifdef CONFIG_ACPI
54363f010cf1STomasz Nowicki
54373f010cf1STomasz Nowicki #define ACPI_GICV3_ITS_MEM_SIZE (SZ_128K)
54383f010cf1STomasz Nowicki
5439d1ce263fSRobert Richter #ifdef CONFIG_ACPI_NUMA
5440dbd2b826SGanapatrao Kulkarni struct its_srat_map {
5441dbd2b826SGanapatrao Kulkarni /* numa node id */
5442dbd2b826SGanapatrao Kulkarni u32 numa_node;
5443dbd2b826SGanapatrao Kulkarni /* GIC ITS ID */
5444dbd2b826SGanapatrao Kulkarni u32 its_id;
5445dbd2b826SGanapatrao Kulkarni };
5446dbd2b826SGanapatrao Kulkarni
5447fdf6e7a8SHanjun Guo static struct its_srat_map *its_srat_maps __initdata;
5448dbd2b826SGanapatrao Kulkarni static int its_in_srat __initdata;
5449dbd2b826SGanapatrao Kulkarni
acpi_get_its_numa_node(u32 its_id)5450dbd2b826SGanapatrao Kulkarni static int __init acpi_get_its_numa_node(u32 its_id)
5451dbd2b826SGanapatrao Kulkarni {
5452dbd2b826SGanapatrao Kulkarni int i;
5453dbd2b826SGanapatrao Kulkarni
5454dbd2b826SGanapatrao Kulkarni for (i = 0; i < its_in_srat; i++) {
5455dbd2b826SGanapatrao Kulkarni if (its_id == its_srat_maps[i].its_id)
5456dbd2b826SGanapatrao Kulkarni return its_srat_maps[i].numa_node;
5457dbd2b826SGanapatrao Kulkarni }
5458dbd2b826SGanapatrao Kulkarni return NUMA_NO_NODE;
5459dbd2b826SGanapatrao Kulkarni }
5460dbd2b826SGanapatrao Kulkarni
gic_acpi_match_srat_its(union acpi_subtable_headers * header,const unsigned long end)546160574d1eSKeith Busch static int __init gic_acpi_match_srat_its(union acpi_subtable_headers *header,
5462fdf6e7a8SHanjun Guo const unsigned long end)
5463fdf6e7a8SHanjun Guo {
5464fdf6e7a8SHanjun Guo return 0;
5465fdf6e7a8SHanjun Guo }
5466fdf6e7a8SHanjun Guo
gic_acpi_parse_srat_its(union acpi_subtable_headers * header,const unsigned long end)546760574d1eSKeith Busch static int __init gic_acpi_parse_srat_its(union acpi_subtable_headers *header,
5468dbd2b826SGanapatrao Kulkarni const unsigned long end)
5469dbd2b826SGanapatrao Kulkarni {
5470dbd2b826SGanapatrao Kulkarni int node;
5471dbd2b826SGanapatrao Kulkarni struct acpi_srat_gic_its_affinity *its_affinity;
5472dbd2b826SGanapatrao Kulkarni
5473dbd2b826SGanapatrao Kulkarni its_affinity = (struct acpi_srat_gic_its_affinity *)header;
5474dbd2b826SGanapatrao Kulkarni if (!its_affinity)
5475dbd2b826SGanapatrao Kulkarni return -EINVAL;
5476dbd2b826SGanapatrao Kulkarni
5477dbd2b826SGanapatrao Kulkarni if (its_affinity->header.length < sizeof(*its_affinity)) {
5478dbd2b826SGanapatrao Kulkarni pr_err("SRAT: Invalid header length %d in ITS affinity\n",
5479dbd2b826SGanapatrao Kulkarni its_affinity->header.length);
5480dbd2b826SGanapatrao Kulkarni return -EINVAL;
5481dbd2b826SGanapatrao Kulkarni }
5482dbd2b826SGanapatrao Kulkarni
548395ac5bf4SJonathan Cameron /*
548495ac5bf4SJonathan Cameron * Note that in theory a new proximity node could be created by this
548595ac5bf4SJonathan Cameron * entry as it is an SRAT resource allocation structure.
548695ac5bf4SJonathan Cameron * We do not currently support doing so.
548795ac5bf4SJonathan Cameron */
548895ac5bf4SJonathan Cameron node = pxm_to_node(its_affinity->proximity_domain);
5489dbd2b826SGanapatrao Kulkarni
5490dbd2b826SGanapatrao Kulkarni if (node == NUMA_NO_NODE || node >= MAX_NUMNODES) {
5491dbd2b826SGanapatrao Kulkarni pr_err("SRAT: Invalid NUMA node %d in ITS affinity\n", node);
5492dbd2b826SGanapatrao Kulkarni return 0;
5493dbd2b826SGanapatrao Kulkarni }
5494dbd2b826SGanapatrao Kulkarni
5495dbd2b826SGanapatrao Kulkarni its_srat_maps[its_in_srat].numa_node = node;
5496dbd2b826SGanapatrao Kulkarni its_srat_maps[its_in_srat].its_id = its_affinity->its_id;
5497dbd2b826SGanapatrao Kulkarni its_in_srat++;
5498dbd2b826SGanapatrao Kulkarni pr_info("SRAT: PXM %d -> ITS %d -> Node %d\n",
5499dbd2b826SGanapatrao Kulkarni its_affinity->proximity_domain, its_affinity->its_id, node);
5500dbd2b826SGanapatrao Kulkarni
5501dbd2b826SGanapatrao Kulkarni return 0;
5502dbd2b826SGanapatrao Kulkarni }
5503dbd2b826SGanapatrao Kulkarni
acpi_table_parse_srat_its(void)5504dbd2b826SGanapatrao Kulkarni static void __init acpi_table_parse_srat_its(void)
5505dbd2b826SGanapatrao Kulkarni {
5506fdf6e7a8SHanjun Guo int count;
5507fdf6e7a8SHanjun Guo
5508fdf6e7a8SHanjun Guo count = acpi_table_parse_entries(ACPI_SIG_SRAT,
5509fdf6e7a8SHanjun Guo sizeof(struct acpi_table_srat),
5510fdf6e7a8SHanjun Guo ACPI_SRAT_TYPE_GIC_ITS_AFFINITY,
5511fdf6e7a8SHanjun Guo gic_acpi_match_srat_its, 0);
5512fdf6e7a8SHanjun Guo if (count <= 0)
5513fdf6e7a8SHanjun Guo return;
5514fdf6e7a8SHanjun Guo
55156da2ec56SKees Cook its_srat_maps = kmalloc_array(count, sizeof(struct its_srat_map),
5516fdf6e7a8SHanjun Guo GFP_KERNEL);
5517944a1a17SZhen Lei if (!its_srat_maps)
5518fdf6e7a8SHanjun Guo return;
5519fdf6e7a8SHanjun Guo
5520dbd2b826SGanapatrao Kulkarni acpi_table_parse_entries(ACPI_SIG_SRAT,
5521dbd2b826SGanapatrao Kulkarni sizeof(struct acpi_table_srat),
5522dbd2b826SGanapatrao Kulkarni ACPI_SRAT_TYPE_GIC_ITS_AFFINITY,
5523dbd2b826SGanapatrao Kulkarni gic_acpi_parse_srat_its, 0);
5524dbd2b826SGanapatrao Kulkarni }
5525fdf6e7a8SHanjun Guo
5526fdf6e7a8SHanjun Guo /* free the its_srat_maps after ITS probing */
acpi_its_srat_maps_free(void)5527fdf6e7a8SHanjun Guo static void __init acpi_its_srat_maps_free(void)
5528fdf6e7a8SHanjun Guo {
5529fdf6e7a8SHanjun Guo kfree(its_srat_maps);
5530fdf6e7a8SHanjun Guo }
5531dbd2b826SGanapatrao Kulkarni #else
acpi_table_parse_srat_its(void)5532dbd2b826SGanapatrao Kulkarni static void __init acpi_table_parse_srat_its(void) { }
acpi_get_its_numa_node(u32 its_id)5533dbd2b826SGanapatrao Kulkarni static int __init acpi_get_its_numa_node(u32 its_id) { return NUMA_NO_NODE; }
acpi_its_srat_maps_free(void)5534fdf6e7a8SHanjun Guo static void __init acpi_its_srat_maps_free(void) { }
5535dbd2b826SGanapatrao Kulkarni #endif
5536dbd2b826SGanapatrao Kulkarni
gic_acpi_parse_madt_its(union acpi_subtable_headers * header,const unsigned long end)553760574d1eSKeith Busch static int __init gic_acpi_parse_madt_its(union acpi_subtable_headers *header,
55383f010cf1STomasz Nowicki const unsigned long end)
55393f010cf1STomasz Nowicki {
55403f010cf1STomasz Nowicki struct acpi_madt_generic_translator *its_entry;
55413f010cf1STomasz Nowicki struct fwnode_handle *dom_handle;
55429585a495SMarc Zyngier struct its_node *its;
55433f010cf1STomasz Nowicki struct resource res;
55443f010cf1STomasz Nowicki int err;
55453f010cf1STomasz Nowicki
55463f010cf1STomasz Nowicki its_entry = (struct acpi_madt_generic_translator *)header;
55473f010cf1STomasz Nowicki memset(&res, 0, sizeof(res));
55483f010cf1STomasz Nowicki res.start = its_entry->base_address;
55493f010cf1STomasz Nowicki res.end = its_entry->base_address + ACPI_GICV3_ITS_MEM_SIZE - 1;
55503f010cf1STomasz Nowicki res.flags = IORESOURCE_MEM;
55513f010cf1STomasz Nowicki
55525778cc77SMarc Zyngier dom_handle = irq_domain_alloc_fwnode(&res.start);
55533f010cf1STomasz Nowicki if (!dom_handle) {
55543f010cf1STomasz Nowicki pr_err("ITS@%pa: Unable to allocate GICv3 ITS domain token\n",
55553f010cf1STomasz Nowicki &res.start);
55563f010cf1STomasz Nowicki return -ENOMEM;
55573f010cf1STomasz Nowicki }
55583f010cf1STomasz Nowicki
55598b4282e6SShameer Kolothum err = iort_register_domain_token(its_entry->translation_id, res.start,
55608b4282e6SShameer Kolothum dom_handle);
55613f010cf1STomasz Nowicki if (err) {
55623f010cf1STomasz Nowicki pr_err("ITS@%pa: Unable to register GICv3 ITS domain token (ITS ID %d) to IORT\n",
55633f010cf1STomasz Nowicki &res.start, its_entry->translation_id);
55643f010cf1STomasz Nowicki goto dom_err;
55653f010cf1STomasz Nowicki }
55663f010cf1STomasz Nowicki
55679585a495SMarc Zyngier its = its_node_init(&res, dom_handle,
5568dbd2b826SGanapatrao Kulkarni acpi_get_its_numa_node(its_entry->translation_id));
55699585a495SMarc Zyngier if (!its) {
55709585a495SMarc Zyngier err = -ENOMEM;
55719585a495SMarc Zyngier goto node_err;
55729585a495SMarc Zyngier }
55739585a495SMarc Zyngier
55749585a495SMarc Zyngier err = its_probe_one(its);
55753f010cf1STomasz Nowicki if (!err)
55763f010cf1STomasz Nowicki return 0;
55773f010cf1STomasz Nowicki
55789585a495SMarc Zyngier node_err:
55793f010cf1STomasz Nowicki iort_deregister_domain_token(its_entry->translation_id);
55803f010cf1STomasz Nowicki dom_err:
55813f010cf1STomasz Nowicki irq_domain_free_fwnode(dom_handle);
55823f010cf1STomasz Nowicki return err;
55833f010cf1STomasz Nowicki }
55843f010cf1STomasz Nowicki
its_acpi_reset(union acpi_subtable_headers * header,const unsigned long end)5585c733ebb7SMarc Zyngier static int __init its_acpi_reset(union acpi_subtable_headers *header,
5586c733ebb7SMarc Zyngier const unsigned long end)
5587c733ebb7SMarc Zyngier {
5588c733ebb7SMarc Zyngier struct acpi_madt_generic_translator *its_entry;
5589c733ebb7SMarc Zyngier struct resource res;
5590c733ebb7SMarc Zyngier
5591c733ebb7SMarc Zyngier its_entry = (struct acpi_madt_generic_translator *)header;
5592c733ebb7SMarc Zyngier res = (struct resource) {
5593c733ebb7SMarc Zyngier .start = its_entry->base_address,
5594c733ebb7SMarc Zyngier .end = its_entry->base_address + ACPI_GICV3_ITS_MEM_SIZE - 1,
5595c733ebb7SMarc Zyngier .flags = IORESOURCE_MEM,
5596c733ebb7SMarc Zyngier };
5597c733ebb7SMarc Zyngier
5598c733ebb7SMarc Zyngier return its_reset_one(&res);
5599c733ebb7SMarc Zyngier }
5600c733ebb7SMarc Zyngier
its_acpi_probe(void)56013f010cf1STomasz Nowicki static void __init its_acpi_probe(void)
56023f010cf1STomasz Nowicki {
5603dbd2b826SGanapatrao Kulkarni acpi_table_parse_srat_its();
5604c733ebb7SMarc Zyngier /*
5605c733ebb7SMarc Zyngier * Make sure *all* the ITS are reset before we probe any, as
5606c733ebb7SMarc Zyngier * they may be sharing memory. If any of the ITS fails to
5607c733ebb7SMarc Zyngier * reset, don't even try to go any further, as this could
5608c733ebb7SMarc Zyngier * result in something even worse.
5609c733ebb7SMarc Zyngier */
5610c733ebb7SMarc Zyngier if (acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_TRANSLATOR,
5611c733ebb7SMarc Zyngier its_acpi_reset, 0) > 0)
56123f010cf1STomasz Nowicki acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_TRANSLATOR,
56133f010cf1STomasz Nowicki gic_acpi_parse_madt_its, 0);
5614fdf6e7a8SHanjun Guo acpi_its_srat_maps_free();
56153f010cf1STomasz Nowicki }
56163f010cf1STomasz Nowicki #else
its_acpi_probe(void)56173f010cf1STomasz Nowicki static void __init its_acpi_probe(void) { }
56183f010cf1STomasz Nowicki #endif
56193f010cf1STomasz Nowicki
its_lpi_memreserve_init(void)5620d23bc2bcSValentin Schneider int __init its_lpi_memreserve_init(void)
5621d23bc2bcSValentin Schneider {
5622d23bc2bcSValentin Schneider int state;
5623d23bc2bcSValentin Schneider
5624d23bc2bcSValentin Schneider if (!efi_enabled(EFI_CONFIG_TABLES))
5625d23bc2bcSValentin Schneider return 0;
5626d23bc2bcSValentin Schneider
5627eba1e44bSMarc Zyngier if (list_empty(&its_nodes))
5628eba1e44bSMarc Zyngier return 0;
5629eba1e44bSMarc Zyngier
5630835f442fSValentin Schneider gic_rdists->cpuhp_memreserve_state = CPUHP_INVALID;
5631d23bc2bcSValentin Schneider state = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN,
5632d23bc2bcSValentin Schneider "irqchip/arm/gicv3/memreserve:online",
5633d23bc2bcSValentin Schneider its_cpu_memreserve_lpi,
5634d23bc2bcSValentin Schneider NULL);
5635d23bc2bcSValentin Schneider if (state < 0)
5636d23bc2bcSValentin Schneider return state;
5637d23bc2bcSValentin Schneider
5638835f442fSValentin Schneider gic_rdists->cpuhp_memreserve_state = state;
5639835f442fSValentin Schneider
5640d23bc2bcSValentin Schneider return 0;
5641d23bc2bcSValentin Schneider }
5642d23bc2bcSValentin Schneider
its_init(struct fwnode_handle * handle,struct rdists * rdists,struct irq_domain * parent_domain)5643db40f0a7STomasz Nowicki int __init its_init(struct fwnode_handle *handle, struct rdists *rdists,
5644db40f0a7STomasz Nowicki struct irq_domain *parent_domain)
5645db40f0a7STomasz Nowicki {
5646db40f0a7STomasz Nowicki struct device_node *of_node;
56478fff27aeSMarc Zyngier struct its_node *its;
56488fff27aeSMarc Zyngier bool has_v4 = false;
56493c40706dSMarc Zyngier bool has_v4_1 = false;
56508fff27aeSMarc Zyngier int err;
5651db40f0a7STomasz Nowicki
56525e516846SMarc Zyngier gic_rdists = rdists;
56535e516846SMarc Zyngier
5654db40f0a7STomasz Nowicki its_parent = parent_domain;
5655db40f0a7STomasz Nowicki of_node = to_of_node(handle);
5656db40f0a7STomasz Nowicki if (of_node)
5657db40f0a7STomasz Nowicki its_of_probe(of_node);
5658db40f0a7STomasz Nowicki else
56593f010cf1STomasz Nowicki its_acpi_probe();
5660db40f0a7STomasz Nowicki
56614c21f3c2SMarc Zyngier if (list_empty(&its_nodes)) {
56624c21f3c2SMarc Zyngier pr_warn("ITS: No ITS available, not enabling LPIs\n");
56634c21f3c2SMarc Zyngier return -ENXIO;
56644c21f3c2SMarc Zyngier }
56654c21f3c2SMarc Zyngier
566611e37d35SMarc Zyngier err = allocate_lpi_tables();
56678fff27aeSMarc Zyngier if (err)
56688fff27aeSMarc Zyngier return err;
56698fff27aeSMarc Zyngier
56703c40706dSMarc Zyngier list_for_each_entry(its, &its_nodes, entry) {
56710dd57fedSMarc Zyngier has_v4 |= is_v4(its);
56723c40706dSMarc Zyngier has_v4_1 |= is_v4_1(its);
56733c40706dSMarc Zyngier }
56743c40706dSMarc Zyngier
56753c40706dSMarc Zyngier /* Don't bother with inconsistent systems */
56763c40706dSMarc Zyngier if (WARN_ON(!has_v4_1 && rdists->has_rvpeid))
56773c40706dSMarc Zyngier rdists->has_rvpeid = false;
56788fff27aeSMarc Zyngier
56798fff27aeSMarc Zyngier if (has_v4 & rdists->has_vlpis) {
5680166cba71SMarc Zyngier const struct irq_domain_ops *sgi_ops;
5681166cba71SMarc Zyngier
5682166cba71SMarc Zyngier if (has_v4_1)
5683166cba71SMarc Zyngier sgi_ops = &its_sgi_domain_ops;
5684166cba71SMarc Zyngier else
5685166cba71SMarc Zyngier sgi_ops = NULL;
5686166cba71SMarc Zyngier
56873d63cb53SMarc Zyngier if (its_init_vpe_domain() ||
5688166cba71SMarc Zyngier its_init_v4(parent_domain, &its_vpe_domain_ops, sgi_ops)) {
56898fff27aeSMarc Zyngier rdists->has_vlpis = false;
56908fff27aeSMarc Zyngier pr_err("ITS: Disabling GICv4 support\n");
56918fff27aeSMarc Zyngier }
56928fff27aeSMarc Zyngier }
56938fff27aeSMarc Zyngier
5694dba0bc7bSDerek Basehore register_syscore_ops(&its_syscore_ops);
5695dba0bc7bSDerek Basehore
56968fff27aeSMarc Zyngier return 0;
56974c21f3c2SMarc Zyngier }
5698