38 #define ctxid2asid(asid)	((asid) & ~ASID_MASK)  argument
39 #define asid2ctxid(asid, genid)	((asid) | (genid))  argument
44 	u32 asid;  in get_cpu_asid_bits()  local
50 		pr_warn("CPU%d: Unknown ASID size (%d); assuming 8-bit\n",  in get_cpu_asid_bits()
54 		asid = 8;  in get_cpu_asid_bits()
57 		asid = 16;  in get_cpu_asid_bits()
60 	return asid;  in get_cpu_asid_bits()
66 	u32 asid = get_cpu_asid_bits();  in verify_cpu_asid_bits()  local
68 	if (asid < asid_bits) {  in verify_cpu_asid_bits()
70 		 * We cannot decrease the ASID size at runtime, so panic if we support  in verify_cpu_asid_bits()
71 		 * fewer ASID bits than the boot CPU.  in verify_cpu_asid_bits()
73 		pr_crit("CPU%d: smaller ASID size(%u) than boot CPU (%u)\n",  in verify_cpu_asid_bits()
74 				smp_processor_id(), asid, asid_bits);  in verify_cpu_asid_bits()
85 	 * is set, then the ASID will map only userspace. Thus  in set_kpti_asid_bits()
101 #define asid_gen_match(asid) \  argument
102 	(!(((asid) ^ atomic64_read(&asid_generation)) >> asid_bits))
107 	u64 asid;  in flush_context()  local
109 	/* Update the list of reserved ASIDs and the ASID bitmap. */  in flush_context()
113 		asid = atomic64_xchg_relaxed(&per_cpu(active_asids, i), 0);  in flush_context()
118 		 * ASID, as this is the only trace we have of  in flush_context()
121 		if (asid == 0)  in flush_context()
122 			asid = per_cpu(reserved_asids, i);  in flush_context()
123 		__set_bit(ctxid2asid(asid), asid_map);  in flush_context()
124 		per_cpu(reserved_asids, i) = asid;  in flush_context()
134 static bool check_update_reserved_asid(u64 asid, u64 newasid)  in check_update_reserved_asid()  argument
142 	 * (i.e. the same ASID in the current generation) but we can't  in check_update_reserved_asid()
144 	 * of the old ASID are updated to reflect the mm. Failure to do  in check_update_reserved_asid()
145 	 * so could result in us missing the reserved ASID in a future  in check_update_reserved_asid()
149 		if (per_cpu(reserved_asids, cpu) == asid) {  in check_update_reserved_asid()
161 	u64 asid = atomic64_read(&mm->context.id);  in new_context()  local
164 	if (asid != 0) {  in new_context()
165 		u64 newasid = asid2ctxid(ctxid2asid(asid), generation);  in new_context()
168 		 * If our current ASID was active during a rollover, we  in new_context()
171 		if (check_update_reserved_asid(asid, newasid))  in new_context()
183 		 * We had a valid ASID in a previous life, so try to re-use  in new_context()
186 		if (!__test_and_set_bit(ctxid2asid(asid), asid_map))  in new_context()
191 	 * Allocate a free ASID. If we can't find one, take a note of the  in new_context()
193 	 * always count from ASID #2 (index 1), as we use ASID #0 when setting  in new_context()
197 	asid = find_next_zero_bit(asid_map, NUM_USER_ASIDS, cur_idx);  in new_context()
198 	if (asid != NUM_USER_ASIDS)  in new_context()
207 	asid = find_next_zero_bit(asid_map, NUM_USER_ASIDS, 1);  in new_context()
210 	__set_bit(asid, asid_map);  in new_context()
211 	cur_idx = asid;  in new_context()
212 	return asid2ctxid(asid, generation);  in new_context()
219 	u64 asid, old_active_asid;  in check_and_switch_context()  local
224 	asid = atomic64_read(&mm->context.id);  in check_and_switch_context()
228 	 * If our active_asids is non-zero and the ASID matches the current  in check_and_switch_context()
236 	 * - We get a valid ASID back from the cmpxchg, which means the  in check_and_switch_context()
241 	if (old_active_asid && asid_gen_match(asid) &&  in check_and_switch_context()
243 				     old_active_asid, asid))  in check_and_switch_context()
247 	/* Check that our ASID belongs to the current generation. */  in check_and_switch_context()
248 	asid = atomic64_read(&mm->context.id);  in check_and_switch_context()
249 	if (!asid_gen_match(asid)) {  in check_and_switch_context()
250 		asid = new_context(mm);  in check_and_switch_context()
251 		atomic64_set(&mm->context.id, asid);  in check_and_switch_context()
258 	atomic64_set(this_cpu_ptr(&active_asids), asid);  in check_and_switch_context()
276 	u64 asid;  in arm64_mm_context_get()  local
283 	asid = atomic64_read(&mm->context.id);  in arm64_mm_context_get()
289 		asid = 0;  in arm64_mm_context_get()
293 	if (!asid_gen_match(asid)) {  in arm64_mm_context_get()
295 		 * We went through one or more rollover since that ASID was  in arm64_mm_context_get()
298 		asid = new_context(mm);  in arm64_mm_context_get()
299 		atomic64_set(&mm->context.id, asid);  in arm64_mm_context_get()
303 	__set_bit(ctxid2asid(asid), pinned_asid_map);  in arm64_mm_context_get()
309 	asid = ctxid2asid(asid);  in arm64_mm_context_get()
312 	if (asid && arm64_kernel_unmapped_at_el0())  in arm64_mm_context_get()
313 		asid |= 1;  in arm64_mm_context_get()
315 	return asid;  in arm64_mm_context_get()
322 	u64 asid = atomic64_read(&mm->context.id);  in arm64_mm_context_put()  local
330 		__clear_bit(ctxid2asid(asid), pinned_asid_map);  in arm64_mm_context_put()
352 	unsigned long asid = ASID(mm);  in cpu_do_switch_mm()  local
355 	/* Skip CNP for the reserved ASID */  in cpu_do_switch_mm()
356 	if (system_supports_cnp() && asid)  in cpu_do_switch_mm()
359 	/* SW PAN needs a copy of the ASID in TTBR0 for entry */  in cpu_do_switch_mm()
361 		ttbr0 |= FIELD_PREP(TTBR_ASID_MASK, asid);  in cpu_do_switch_mm()
363 	/* Set ASID in TTBR1 since TCR.A1 is set */  in cpu_do_switch_mm()
365 	ttbr1 |= FIELD_PREP(TTBR_ASID_MASK, asid);  in cpu_do_switch_mm()
385 	 * one more ASID than CPUs. ASID #0 is reserved for init_mm.  in asids_update_limit()
388 	pr_info("ASID allocator initialised with %lu entries\n",  in asids_update_limit()
392 	 * There must always be an ASID available after rollover. Ensure that,  in asids_update_limit()
393 	 * even if all CPUs have a reserved ASID and the maximum number of ASIDs  in asids_update_limit()
394 	 * are pinned, there still is at least one empty slot in the ASID map.  in asids_update_limit()
416 	 * and reserve kernel ASID's from beginning.  in asids_init()