1b3901d54SCatalin Marinas /* 2b3901d54SCatalin Marinas * Based on arch/arm/mm/context.c 3b3901d54SCatalin Marinas * 4b3901d54SCatalin Marinas * Copyright (C) 2002-2003 Deep Blue Solutions Ltd, all rights reserved. 5b3901d54SCatalin Marinas * Copyright (C) 2012 ARM Ltd. 6b3901d54SCatalin Marinas * 7b3901d54SCatalin Marinas * This program is free software; you can redistribute it and/or modify 8b3901d54SCatalin Marinas * it under the terms of the GNU General Public License version 2 as 9b3901d54SCatalin Marinas * published by the Free Software Foundation. 10b3901d54SCatalin Marinas * 11b3901d54SCatalin Marinas * This program is distributed in the hope that it will be useful, 12b3901d54SCatalin Marinas * but WITHOUT ANY WARRANTY; without even the implied warranty of 13b3901d54SCatalin Marinas * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14b3901d54SCatalin Marinas * GNU General Public License for more details. 15b3901d54SCatalin Marinas * 16b3901d54SCatalin Marinas * You should have received a copy of the GNU General Public License 17b3901d54SCatalin Marinas * along with this program. If not, see <http://www.gnu.org/licenses/>. 18b3901d54SCatalin Marinas */ 19b3901d54SCatalin Marinas 205aec715dSWill Deacon #include <linux/bitops.h> 21b3901d54SCatalin Marinas #include <linux/sched.h> 225aec715dSWill Deacon #include <linux/slab.h> 23b3901d54SCatalin Marinas #include <linux/mm.h> 24b3901d54SCatalin Marinas 255aec715dSWill Deacon #include <asm/cpufeature.h> 26b3901d54SCatalin Marinas #include <asm/mmu_context.h> 27b3901d54SCatalin Marinas #include <asm/tlbflush.h> 28b3901d54SCatalin Marinas 295aec715dSWill Deacon static u32 asid_bits; 30b3901d54SCatalin Marinas static DEFINE_RAW_SPINLOCK(cpu_asid_lock); 315aec715dSWill Deacon 325aec715dSWill Deacon static atomic64_t asid_generation; 335aec715dSWill Deacon static unsigned long *asid_map; 345aec715dSWill Deacon 355aec715dSWill Deacon static DEFINE_PER_CPU(atomic64_t, active_asids); 365aec715dSWill Deacon static DEFINE_PER_CPU(u64, reserved_asids); 375aec715dSWill Deacon static cpumask_t tlb_flush_pending; 385aec715dSWill Deacon 395aec715dSWill Deacon #define ASID_MASK (~GENMASK(asid_bits - 1, 0)) 405aec715dSWill Deacon #define ASID_FIRST_VERSION (1UL << asid_bits) 415aec715dSWill Deacon #define NUM_USER_ASIDS ASID_FIRST_VERSION 425aec715dSWill Deacon 435aec715dSWill Deacon static void flush_context(unsigned int cpu) 445aec715dSWill Deacon { 455aec715dSWill Deacon int i; 465aec715dSWill Deacon u64 asid; 475aec715dSWill Deacon 485aec715dSWill Deacon /* Update the list of reserved ASIDs and the ASID bitmap. */ 495aec715dSWill Deacon bitmap_clear(asid_map, 0, NUM_USER_ASIDS); 50b3901d54SCatalin Marinas 51b3901d54SCatalin Marinas /* 525aec715dSWill Deacon * Ensure the generation bump is observed before we xchg the 535aec715dSWill Deacon * active_asids. 54b3901d54SCatalin Marinas */ 555aec715dSWill Deacon smp_wmb(); 565aec715dSWill Deacon 575aec715dSWill Deacon for_each_possible_cpu(i) { 585aec715dSWill Deacon asid = atomic64_xchg_relaxed(&per_cpu(active_asids, i), 0); 595aec715dSWill Deacon /* 605aec715dSWill Deacon * If this CPU has already been through a 615aec715dSWill Deacon * rollover, but hasn't run another task in 625aec715dSWill Deacon * the meantime, we must preserve its reserved 635aec715dSWill Deacon * ASID, as this is the only trace we have of 645aec715dSWill Deacon * the process it is still running. 655aec715dSWill Deacon */ 665aec715dSWill Deacon if (asid == 0) 675aec715dSWill Deacon asid = per_cpu(reserved_asids, i); 685aec715dSWill Deacon __set_bit(asid & ~ASID_MASK, asid_map); 695aec715dSWill Deacon per_cpu(reserved_asids, i) = asid; 70b3901d54SCatalin Marinas } 71b3901d54SCatalin Marinas 725aec715dSWill Deacon /* Queue a TLB invalidate and flush the I-cache if necessary. */ 735aec715dSWill Deacon cpumask_setall(&tlb_flush_pending); 745aec715dSWill Deacon 75b3901d54SCatalin Marinas if (icache_is_aivivt()) 765aec715dSWill Deacon __flush_icache_all(); 77b3901d54SCatalin Marinas } 78b3901d54SCatalin Marinas 795aec715dSWill Deacon static int is_reserved_asid(u64 asid) 805aec715dSWill Deacon { 815aec715dSWill Deacon int cpu; 825aec715dSWill Deacon for_each_possible_cpu(cpu) 835aec715dSWill Deacon if (per_cpu(reserved_asids, cpu) == asid) 845aec715dSWill Deacon return 1; 855aec715dSWill Deacon return 0; 865aec715dSWill Deacon } 875aec715dSWill Deacon 885aec715dSWill Deacon static u64 new_context(struct mm_struct *mm, unsigned int cpu) 895aec715dSWill Deacon { 905aec715dSWill Deacon static u32 cur_idx = 1; 915aec715dSWill Deacon u64 asid = atomic64_read(&mm->context.id); 925aec715dSWill Deacon u64 generation = atomic64_read(&asid_generation); 935aec715dSWill Deacon 945aec715dSWill Deacon if (asid != 0) { 955aec715dSWill Deacon /* 965aec715dSWill Deacon * If our current ASID was active during a rollover, we 975aec715dSWill Deacon * can continue to use it and this was just a false alarm. 985aec715dSWill Deacon */ 995aec715dSWill Deacon if (is_reserved_asid(asid)) 1005aec715dSWill Deacon return generation | (asid & ~ASID_MASK); 1015aec715dSWill Deacon 1025aec715dSWill Deacon /* 1035aec715dSWill Deacon * We had a valid ASID in a previous life, so try to re-use 1045aec715dSWill Deacon * it if possible. 1055aec715dSWill Deacon */ 1065aec715dSWill Deacon asid &= ~ASID_MASK; 1075aec715dSWill Deacon if (!__test_and_set_bit(asid, asid_map)) 1085aec715dSWill Deacon goto bump_gen; 1095aec715dSWill Deacon } 1105aec715dSWill Deacon 1115aec715dSWill Deacon /* 1125aec715dSWill Deacon * Allocate a free ASID. If we can't find one, take a note of the 1135aec715dSWill Deacon * currently active ASIDs and mark the TLBs as requiring flushes. 1145aec715dSWill Deacon * We always count from ASID #1, as we use ASID #0 when setting a 1155aec715dSWill Deacon * reserved TTBR0 for the init_mm. 1165aec715dSWill Deacon */ 1175aec715dSWill Deacon asid = find_next_zero_bit(asid_map, NUM_USER_ASIDS, cur_idx); 1185aec715dSWill Deacon if (asid != NUM_USER_ASIDS) 1195aec715dSWill Deacon goto set_asid; 1205aec715dSWill Deacon 1215aec715dSWill Deacon /* We're out of ASIDs, so increment the global generation count */ 1225aec715dSWill Deacon generation = atomic64_add_return_relaxed(ASID_FIRST_VERSION, 1235aec715dSWill Deacon &asid_generation); 1245aec715dSWill Deacon flush_context(cpu); 1255aec715dSWill Deacon 1265aec715dSWill Deacon /* We have at least 1 ASID per CPU, so this will always succeed */ 1275aec715dSWill Deacon asid = find_next_zero_bit(asid_map, NUM_USER_ASIDS, 1); 1285aec715dSWill Deacon 1295aec715dSWill Deacon set_asid: 1305aec715dSWill Deacon __set_bit(asid, asid_map); 1315aec715dSWill Deacon cur_idx = asid; 1325aec715dSWill Deacon 1335aec715dSWill Deacon bump_gen: 1345aec715dSWill Deacon asid |= generation; 1355aec715dSWill Deacon cpumask_clear(mm_cpumask(mm)); 1365aec715dSWill Deacon return asid; 1375aec715dSWill Deacon } 1385aec715dSWill Deacon 1395aec715dSWill Deacon void check_and_switch_context(struct mm_struct *mm, unsigned int cpu) 140b3901d54SCatalin Marinas { 141b3901d54SCatalin Marinas unsigned long flags; 1425aec715dSWill Deacon u64 asid; 1435aec715dSWill Deacon 1445aec715dSWill Deacon asid = atomic64_read(&mm->context.id); 145b3901d54SCatalin Marinas 146b3901d54SCatalin Marinas /* 1475aec715dSWill Deacon * The memory ordering here is subtle. We rely on the control 1485aec715dSWill Deacon * dependency between the generation read and the update of 1495aec715dSWill Deacon * active_asids to ensure that we are synchronised with a 1505aec715dSWill Deacon * parallel rollover (i.e. this pairs with the smp_wmb() in 1515aec715dSWill Deacon * flush_context). 152b3901d54SCatalin Marinas */ 1535aec715dSWill Deacon if (!((asid ^ atomic64_read(&asid_generation)) >> asid_bits) 1545aec715dSWill Deacon && atomic64_xchg_relaxed(&per_cpu(active_asids, cpu), asid)) 1555aec715dSWill Deacon goto switch_mm_fastpath; 156b3901d54SCatalin Marinas 1575aec715dSWill Deacon raw_spin_lock_irqsave(&cpu_asid_lock, flags); 1585aec715dSWill Deacon /* Check that our ASID belongs to the current generation. */ 1595aec715dSWill Deacon asid = atomic64_read(&mm->context.id); 1605aec715dSWill Deacon if ((asid ^ atomic64_read(&asid_generation)) >> asid_bits) { 1615aec715dSWill Deacon asid = new_context(mm, cpu); 1625aec715dSWill Deacon atomic64_set(&mm->context.id, asid); 163b3901d54SCatalin Marinas } 164b3901d54SCatalin Marinas 1655aec715dSWill Deacon if (cpumask_test_and_clear_cpu(cpu, &tlb_flush_pending)) 1665aec715dSWill Deacon local_flush_tlb_all(); 167b3901d54SCatalin Marinas 1685aec715dSWill Deacon atomic64_set(&per_cpu(active_asids, cpu), asid); 1695aec715dSWill Deacon cpumask_set_cpu(cpu, mm_cpumask(mm)); 1705aec715dSWill Deacon raw_spin_unlock_irqrestore(&cpu_asid_lock, flags); 171565630d5SCatalin Marinas 1725aec715dSWill Deacon switch_mm_fastpath: 173b3901d54SCatalin Marinas cpu_switch_mm(mm->pgd, mm); 174b3901d54SCatalin Marinas } 175b3901d54SCatalin Marinas 1765aec715dSWill Deacon static int asids_init(void) 177b3901d54SCatalin Marinas { 1785aec715dSWill Deacon int fld = cpuid_feature_extract_field(read_cpuid(ID_AA64MMFR0_EL1), 4); 179b3901d54SCatalin Marinas 1805aec715dSWill Deacon switch (fld) { 1815aec715dSWill Deacon default: 1825aec715dSWill Deacon pr_warn("Unknown ASID size (%d); assuming 8-bit\n", fld); 1835aec715dSWill Deacon /* Fallthrough */ 1845aec715dSWill Deacon case 0: 1855aec715dSWill Deacon asid_bits = 8; 1865aec715dSWill Deacon break; 1875aec715dSWill Deacon case 2: 1885aec715dSWill Deacon asid_bits = 16; 189b3901d54SCatalin Marinas } 190b3901d54SCatalin Marinas 1915aec715dSWill Deacon /* If we end up with more CPUs than ASIDs, expect things to crash */ 1925aec715dSWill Deacon WARN_ON(NUM_USER_ASIDS < num_possible_cpus()); 1935aec715dSWill Deacon atomic64_set(&asid_generation, ASID_FIRST_VERSION); 1945aec715dSWill Deacon asid_map = kzalloc(BITS_TO_LONGS(NUM_USER_ASIDS) * sizeof(*asid_map), 1955aec715dSWill Deacon GFP_KERNEL); 1965aec715dSWill Deacon if (!asid_map) 1975aec715dSWill Deacon panic("Failed to allocate bitmap for %lu ASIDs\n", 1985aec715dSWill Deacon NUM_USER_ASIDS); 1995aec715dSWill Deacon 2005aec715dSWill Deacon pr_info("ASID allocator initialised with %lu entries\n", NUM_USER_ASIDS); 2015aec715dSWill Deacon return 0; 202b3901d54SCatalin Marinas } 2035aec715dSWill Deacon early_initcall(asids_init); 204