1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Generic ASID allocator. 4 * 5 * Based on arch/arm/mm/context.c 6 * 7 * Copyright (C) 2002-2003 Deep Blue Solutions Ltd, all rights reserved. 8 * Copyright (C) 2012 ARM Ltd. 9 */ 10 11 #include <linux/slab.h> 12 #include <linux/mm_types.h> 13 14 #include <asm/asid.h> 15 16 #define reserved_asid(info, cpu) *per_cpu_ptr((info)->reserved, cpu) 17 18 #define ASID_MASK(info) (~GENMASK((info)->bits - 1, 0)) 19 #define ASID_FIRST_VERSION(info) (1UL << ((info)->bits)) 20 21 #define asid2idx(info, asid) (((asid) & ~ASID_MASK(info)) >> (info)->ctxt_shift) 22 #define idx2asid(info, idx) (((idx) << (info)->ctxt_shift) & ~ASID_MASK(info)) 23 24 static void flush_context(struct asid_info *info) 25 { 26 int i; 27 u64 asid; 28 29 /* Update the list of reserved ASIDs and the ASID bitmap. */ 30 bitmap_zero(info->map, NUM_CTXT_ASIDS(info)); 31 32 for_each_possible_cpu(i) { 33 asid = atomic64_xchg_relaxed(&active_asid(info, i), 0); 34 /* 35 * If this CPU has already been through a 36 * rollover, but hasn't run another task in 37 * the meantime, we must preserve its reserved 38 * ASID, as this is the only trace we have of 39 * the process it is still running. 40 */ 41 if (asid == 0) 42 asid = reserved_asid(info, i); 43 __set_bit(asid2idx(info, asid), info->map); 44 reserved_asid(info, i) = asid; 45 } 46 47 /* 48 * Queue a TLB invalidation for each CPU to perform on next 49 * context-switch 50 */ 51 cpumask_setall(&info->flush_pending); 52 } 53 54 static bool check_update_reserved_asid(struct asid_info *info, u64 asid, 55 u64 newasid) 56 { 57 int cpu; 58 bool hit = false; 59 60 /* 61 * Iterate over the set of reserved ASIDs looking for a match. 62 * If we find one, then we can update our mm to use newasid 63 * (i.e. the same ASID in the current generation) but we can't 64 * exit the loop early, since we need to ensure that all copies 65 * of the old ASID are updated to reflect the mm. Failure to do 66 * so could result in us missing the reserved ASID in a future 67 * generation. 68 */ 69 for_each_possible_cpu(cpu) { 70 if (reserved_asid(info, cpu) == asid) { 71 hit = true; 72 reserved_asid(info, cpu) = newasid; 73 } 74 } 75 76 return hit; 77 } 78 79 static u64 new_context(struct asid_info *info, atomic64_t *pasid, 80 struct mm_struct *mm) 81 { 82 static u32 cur_idx = 1; 83 u64 asid = atomic64_read(pasid); 84 u64 generation = atomic64_read(&info->generation); 85 86 if (asid != 0) { 87 u64 newasid = generation | (asid & ~ASID_MASK(info)); 88 89 /* 90 * If our current ASID was active during a rollover, we 91 * can continue to use it and this was just a false alarm. 92 */ 93 if (check_update_reserved_asid(info, asid, newasid)) 94 return newasid; 95 96 /* 97 * We had a valid ASID in a previous life, so try to re-use 98 * it if possible. 99 */ 100 if (!__test_and_set_bit(asid2idx(info, asid), info->map)) 101 return newasid; 102 } 103 104 /* 105 * Allocate a free ASID. If we can't find one, take a note of the 106 * currently active ASIDs and mark the TLBs as requiring flushes. We 107 * always count from ASID #2 (index 1), as we use ASID #0 when setting 108 * a reserved TTBR0 for the init_mm and we allocate ASIDs in even/odd 109 * pairs. 110 */ 111 asid = find_next_zero_bit(info->map, NUM_CTXT_ASIDS(info), cur_idx); 112 if (asid != NUM_CTXT_ASIDS(info)) 113 goto set_asid; 114 115 /* We're out of ASIDs, so increment the global generation count */ 116 generation = atomic64_add_return_relaxed(ASID_FIRST_VERSION(info), 117 &info->generation); 118 flush_context(info); 119 120 /* We have more ASIDs than CPUs, so this will always succeed */ 121 asid = find_next_zero_bit(info->map, NUM_CTXT_ASIDS(info), 1); 122 123 set_asid: 124 __set_bit(asid, info->map); 125 cur_idx = asid; 126 cpumask_clear(mm_cpumask(mm)); 127 return idx2asid(info, asid) | generation; 128 } 129 130 /* 131 * Generate a new ASID for the context. 132 * 133 * @pasid: Pointer to the current ASID batch allocated. It will be updated 134 * with the new ASID batch. 135 * @cpu: current CPU ID. Must have been acquired through get_cpu() 136 */ 137 void asid_new_context(struct asid_info *info, atomic64_t *pasid, 138 unsigned int cpu, struct mm_struct *mm) 139 { 140 unsigned long flags; 141 u64 asid; 142 143 raw_spin_lock_irqsave(&info->lock, flags); 144 /* Check that our ASID belongs to the current generation. */ 145 asid = atomic64_read(pasid); 146 if ((asid ^ atomic64_read(&info->generation)) >> info->bits) { 147 asid = new_context(info, pasid, mm); 148 atomic64_set(pasid, asid); 149 } 150 151 if (cpumask_test_and_clear_cpu(cpu, &info->flush_pending)) 152 info->flush_cpu_ctxt_cb(); 153 154 atomic64_set(&active_asid(info, cpu), asid); 155 cpumask_set_cpu(cpu, mm_cpumask(mm)); 156 raw_spin_unlock_irqrestore(&info->lock, flags); 157 } 158 159 /* 160 * Initialize the ASID allocator 161 * 162 * @info: Pointer to the asid allocator structure 163 * @bits: Number of ASIDs available 164 * @asid_per_ctxt: Number of ASIDs to allocate per-context. ASIDs are 165 * allocated contiguously for a given context. This value should be a power of 166 * 2. 167 */ 168 int asid_allocator_init(struct asid_info *info, 169 u32 bits, unsigned int asid_per_ctxt, 170 void (*flush_cpu_ctxt_cb)(void)) 171 { 172 info->bits = bits; 173 info->ctxt_shift = ilog2(asid_per_ctxt); 174 info->flush_cpu_ctxt_cb = flush_cpu_ctxt_cb; 175 /* 176 * Expect allocation after rollover to fail if we don't have at least 177 * one more ASID than CPUs. ASID #0 is always reserved. 178 */ 179 WARN_ON(NUM_CTXT_ASIDS(info) - 1 <= num_possible_cpus()); 180 atomic64_set(&info->generation, ASID_FIRST_VERSION(info)); 181 info->map = bitmap_zalloc(NUM_CTXT_ASIDS(info), GFP_KERNEL); 182 if (!info->map) 183 return -ENOMEM; 184 185 raw_spin_lock_init(&info->lock); 186 187 return 0; 188 } 189