19a0deeccSJack Steiner /* 29a0deeccSJack Steiner * SN Platform GRU Driver 39a0deeccSJack Steiner * 49a0deeccSJack Steiner * DRIVER TABLE MANAGER + GRU CONTEXT LOAD/UNLOAD 59a0deeccSJack Steiner * 69a0deeccSJack Steiner * Copyright (c) 2008 Silicon Graphics, Inc. All Rights Reserved. 78820f27aSJack Steiner * 88820f27aSJack Steiner * This program is free software; you can redistribute it and/or modify 98820f27aSJack Steiner * it under the terms of the GNU General Public License as published by 108820f27aSJack Steiner * the Free Software Foundation; either version 2 of the License, or 118820f27aSJack Steiner * (at your option) any later version. 128820f27aSJack Steiner * 138820f27aSJack Steiner * This program is distributed in the hope that it will be useful, 148820f27aSJack Steiner * but WITHOUT ANY WARRANTY; without even the implied warranty of 158820f27aSJack Steiner * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 168820f27aSJack Steiner * GNU General Public License for more details. 178820f27aSJack Steiner * 188820f27aSJack Steiner * You should have received a copy of the GNU General Public License 198820f27aSJack Steiner * along with this program; if not, write to the Free Software 208820f27aSJack Steiner * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 219a0deeccSJack Steiner */ 229a0deeccSJack Steiner 239a0deeccSJack Steiner #include <linux/kernel.h> 249a0deeccSJack Steiner #include <linux/slab.h> 259a0deeccSJack Steiner #include <linux/mm.h> 269a0deeccSJack Steiner #include <linux/spinlock.h> 279a0deeccSJack Steiner #include <linux/sched.h> 289a0deeccSJack Steiner #include <linux/device.h> 299a0deeccSJack Steiner #include <linux/list.h> 307f2251b1SJack Steiner #include <linux/err.h> 319a0deeccSJack Steiner #include <asm/uv/uv_hub.h> 329a0deeccSJack Steiner #include "gru.h" 339a0deeccSJack Steiner #include "grutables.h" 349a0deeccSJack Steiner #include "gruhandles.h" 359a0deeccSJack Steiner 369ca8e40cSJack Steiner unsigned long gru_options __read_mostly; 379a0deeccSJack Steiner 389a0deeccSJack Steiner static struct device_driver gru_driver = { 399a0deeccSJack Steiner .name = "gru" 409a0deeccSJack Steiner }; 419a0deeccSJack Steiner 429a0deeccSJack Steiner static struct device gru_device = { 43bb0dc43eSKay Sievers .init_name = "", 449a0deeccSJack Steiner .driver = &gru_driver, 459a0deeccSJack Steiner }; 469a0deeccSJack Steiner 479a0deeccSJack Steiner struct device *grudev = &gru_device; 489a0deeccSJack Steiner 499a0deeccSJack Steiner /* 509a0deeccSJack Steiner * Select a gru fault map to be used by the current cpu. Note that 519a0deeccSJack Steiner * multiple cpus may be using the same map. 529a0deeccSJack Steiner * ZZZ should be inline but did not work on emulator 539a0deeccSJack Steiner */ 549a0deeccSJack Steiner int gru_cpu_fault_map_id(void) 559a0deeccSJack Steiner { 5641314790SJack Steiner #ifdef CONFIG_IA64 5741314790SJack Steiner return uv_blade_processor_id() % GRU_NUM_TFM; 5841314790SJack Steiner #else 594107e1d3SJack Steiner int cpu = smp_processor_id(); 604107e1d3SJack Steiner int id, core; 614107e1d3SJack Steiner 624107e1d3SJack Steiner core = uv_cpu_core_number(cpu); 634107e1d3SJack Steiner id = core + UV_MAX_INT_CORES * uv_cpu_socket_number(cpu); 644107e1d3SJack Steiner return id; 6541314790SJack Steiner #endif 669a0deeccSJack Steiner } 679a0deeccSJack Steiner 689a0deeccSJack Steiner /*--------- ASID Management ------------------------------------------- 699a0deeccSJack Steiner * 709a0deeccSJack Steiner * Initially, assign asids sequentially from MIN_ASID .. MAX_ASID. 719a0deeccSJack Steiner * Once MAX is reached, flush the TLB & start over. However, 729a0deeccSJack Steiner * some asids may still be in use. There won't be many (percentage wise) still 739a0deeccSJack Steiner * in use. Search active contexts & determine the value of the first 749a0deeccSJack Steiner * asid in use ("x"s below). Set "limit" to this value. 759a0deeccSJack Steiner * This defines a block of assignable asids. 769a0deeccSJack Steiner * 779a0deeccSJack Steiner * When "limit" is reached, search forward from limit+1 and determine the 789a0deeccSJack Steiner * next block of assignable asids. 799a0deeccSJack Steiner * 809a0deeccSJack Steiner * Repeat until MAX_ASID is reached, then start over again. 819a0deeccSJack Steiner * 829a0deeccSJack Steiner * Each time MAX_ASID is reached, increment the asid generation. Since 839a0deeccSJack Steiner * the search for in-use asids only checks contexts with GRUs currently 849a0deeccSJack Steiner * assigned, asids in some contexts will be missed. Prior to loading 859a0deeccSJack Steiner * a context, the asid generation of the GTS asid is rechecked. If it 869a0deeccSJack Steiner * doesn't match the current generation, a new asid will be assigned. 879a0deeccSJack Steiner * 889a0deeccSJack Steiner * 0---------------x------------x---------------------x----| 899a0deeccSJack Steiner * ^-next ^-limit ^-MAX_ASID 909a0deeccSJack Steiner * 919a0deeccSJack Steiner * All asid manipulation & context loading/unloading is protected by the 929a0deeccSJack Steiner * gs_lock. 939a0deeccSJack Steiner */ 949a0deeccSJack Steiner 959a0deeccSJack Steiner /* Hit the asid limit. Start over */ 969a0deeccSJack Steiner static int gru_wrap_asid(struct gru_state *gru) 979a0deeccSJack Steiner { 9843884604SJack Steiner gru_dbg(grudev, "gid %d\n", gru->gs_gid); 999a0deeccSJack Steiner STAT(asid_wrap); 1009a0deeccSJack Steiner gru->gs_asid_gen++; 1019a0deeccSJack Steiner return MIN_ASID; 1029a0deeccSJack Steiner } 1039a0deeccSJack Steiner 1049a0deeccSJack Steiner /* Find the next chunk of unused asids */ 1059a0deeccSJack Steiner static int gru_reset_asid_limit(struct gru_state *gru, int asid) 1069a0deeccSJack Steiner { 1079a0deeccSJack Steiner int i, gid, inuse_asid, limit; 1089a0deeccSJack Steiner 10943884604SJack Steiner gru_dbg(grudev, "gid %d, asid 0x%x\n", gru->gs_gid, asid); 1109a0deeccSJack Steiner STAT(asid_next); 1119a0deeccSJack Steiner limit = MAX_ASID; 1129a0deeccSJack Steiner if (asid >= limit) 1139a0deeccSJack Steiner asid = gru_wrap_asid(gru); 11487419412SJack Steiner gru_flush_all_tlb(gru); 1159a0deeccSJack Steiner gid = gru->gs_gid; 1169a0deeccSJack Steiner again: 1179a0deeccSJack Steiner for (i = 0; i < GRU_NUM_CCH; i++) { 118836ce679SJack Steiner if (!gru->gs_gts[i] || is_kernel_context(gru->gs_gts[i])) 1199a0deeccSJack Steiner continue; 1209a0deeccSJack Steiner inuse_asid = gru->gs_gts[i]->ts_gms->ms_asids[gid].mt_asid; 12143884604SJack Steiner gru_dbg(grudev, "gid %d, gts %p, gms %p, inuse 0x%x, cxt %d\n", 12243884604SJack Steiner gru->gs_gid, gru->gs_gts[i], gru->gs_gts[i]->ts_gms, 12343884604SJack Steiner inuse_asid, i); 1249a0deeccSJack Steiner if (inuse_asid == asid) { 1259a0deeccSJack Steiner asid += ASID_INC; 1269a0deeccSJack Steiner if (asid >= limit) { 1279a0deeccSJack Steiner /* 1289a0deeccSJack Steiner * empty range: reset the range limit and 1299a0deeccSJack Steiner * start over 1309a0deeccSJack Steiner */ 1319a0deeccSJack Steiner limit = MAX_ASID; 1329a0deeccSJack Steiner if (asid >= MAX_ASID) 1339a0deeccSJack Steiner asid = gru_wrap_asid(gru); 1349a0deeccSJack Steiner goto again; 1359a0deeccSJack Steiner } 1369a0deeccSJack Steiner } 1379a0deeccSJack Steiner 1389a0deeccSJack Steiner if ((inuse_asid > asid) && (inuse_asid < limit)) 1399a0deeccSJack Steiner limit = inuse_asid; 1409a0deeccSJack Steiner } 1419a0deeccSJack Steiner gru->gs_asid_limit = limit; 1429a0deeccSJack Steiner gru->gs_asid = asid; 14343884604SJack Steiner gru_dbg(grudev, "gid %d, new asid 0x%x, new_limit 0x%x\n", gru->gs_gid, 14443884604SJack Steiner asid, limit); 1459a0deeccSJack Steiner return asid; 1469a0deeccSJack Steiner } 1479a0deeccSJack Steiner 1489a0deeccSJack Steiner /* Assign a new ASID to a thread context. */ 1499a0deeccSJack Steiner static int gru_assign_asid(struct gru_state *gru) 1509a0deeccSJack Steiner { 1519a0deeccSJack Steiner int asid; 1529a0deeccSJack Steiner 1539a0deeccSJack Steiner gru->gs_asid += ASID_INC; 1549a0deeccSJack Steiner asid = gru->gs_asid; 1559a0deeccSJack Steiner if (asid >= gru->gs_asid_limit) 1569a0deeccSJack Steiner asid = gru_reset_asid_limit(gru, asid); 1579a0deeccSJack Steiner 15843884604SJack Steiner gru_dbg(grudev, "gid %d, asid 0x%x\n", gru->gs_gid, asid); 1599a0deeccSJack Steiner return asid; 1609a0deeccSJack Steiner } 1619a0deeccSJack Steiner 1629a0deeccSJack Steiner /* 1639a0deeccSJack Steiner * Clear n bits in a word. Return a word indicating the bits that were cleared. 1649a0deeccSJack Steiner * Optionally, build an array of chars that contain the bit numbers allocated. 1659a0deeccSJack Steiner */ 1669a0deeccSJack Steiner static unsigned long reserve_resources(unsigned long *p, int n, int mmax, 1679a0deeccSJack Steiner char *idx) 1689a0deeccSJack Steiner { 1699a0deeccSJack Steiner unsigned long bits = 0; 1709a0deeccSJack Steiner int i; 1719a0deeccSJack Steiner 1723eac2e95SJack Steiner while (n--) { 1739a0deeccSJack Steiner i = find_first_bit(p, mmax); 1749a0deeccSJack Steiner if (i == mmax) 1759a0deeccSJack Steiner BUG(); 1769a0deeccSJack Steiner __clear_bit(i, p); 1779a0deeccSJack Steiner __set_bit(i, &bits); 1789a0deeccSJack Steiner if (idx) 1799a0deeccSJack Steiner *idx++ = i; 1803eac2e95SJack Steiner } 1819a0deeccSJack Steiner return bits; 1829a0deeccSJack Steiner } 1839a0deeccSJack Steiner 1849ca8e40cSJack Steiner unsigned long gru_reserve_cb_resources(struct gru_state *gru, int cbr_au_count, 1859a0deeccSJack Steiner char *cbmap) 1869a0deeccSJack Steiner { 1879a0deeccSJack Steiner return reserve_resources(&gru->gs_cbr_map, cbr_au_count, GRU_CBR_AU, 1889a0deeccSJack Steiner cbmap); 1899a0deeccSJack Steiner } 1909a0deeccSJack Steiner 1919ca8e40cSJack Steiner unsigned long gru_reserve_ds_resources(struct gru_state *gru, int dsr_au_count, 1929a0deeccSJack Steiner char *dsmap) 1939a0deeccSJack Steiner { 1949a0deeccSJack Steiner return reserve_resources(&gru->gs_dsr_map, dsr_au_count, GRU_DSR_AU, 1959a0deeccSJack Steiner dsmap); 1969a0deeccSJack Steiner } 1979a0deeccSJack Steiner 1989a0deeccSJack Steiner static void reserve_gru_resources(struct gru_state *gru, 1999a0deeccSJack Steiner struct gru_thread_state *gts) 2009a0deeccSJack Steiner { 2019a0deeccSJack Steiner gru->gs_active_contexts++; 2029a0deeccSJack Steiner gts->ts_cbr_map = 2039ca8e40cSJack Steiner gru_reserve_cb_resources(gru, gts->ts_cbr_au_count, 2049a0deeccSJack Steiner gts->ts_cbr_idx); 2059a0deeccSJack Steiner gts->ts_dsr_map = 2069ca8e40cSJack Steiner gru_reserve_ds_resources(gru, gts->ts_dsr_au_count, NULL); 2079a0deeccSJack Steiner } 2089a0deeccSJack Steiner 2099a0deeccSJack Steiner static void free_gru_resources(struct gru_state *gru, 2109a0deeccSJack Steiner struct gru_thread_state *gts) 2119a0deeccSJack Steiner { 2129a0deeccSJack Steiner gru->gs_active_contexts--; 2139a0deeccSJack Steiner gru->gs_cbr_map |= gts->ts_cbr_map; 2149a0deeccSJack Steiner gru->gs_dsr_map |= gts->ts_dsr_map; 2159a0deeccSJack Steiner } 2169a0deeccSJack Steiner 2179a0deeccSJack Steiner /* 2189a0deeccSJack Steiner * Check if a GRU has sufficient free resources to satisfy an allocation 2199a0deeccSJack Steiner * request. Note: GRU locks may or may not be held when this is called. If 2209a0deeccSJack Steiner * not held, recheck after acquiring the appropriate locks. 2219a0deeccSJack Steiner * 2229a0deeccSJack Steiner * Returns 1 if sufficient resources, 0 if not 2239a0deeccSJack Steiner */ 2249a0deeccSJack Steiner static int check_gru_resources(struct gru_state *gru, int cbr_au_count, 2259a0deeccSJack Steiner int dsr_au_count, int max_active_contexts) 2269a0deeccSJack Steiner { 2279a0deeccSJack Steiner return hweight64(gru->gs_cbr_map) >= cbr_au_count 2289a0deeccSJack Steiner && hweight64(gru->gs_dsr_map) >= dsr_au_count 2299a0deeccSJack Steiner && gru->gs_active_contexts < max_active_contexts; 2309a0deeccSJack Steiner } 2319a0deeccSJack Steiner 2329a0deeccSJack Steiner /* 2339a0deeccSJack Steiner * TLB manangment requires tracking all GRU chiplets that have loaded a GSEG 2349a0deeccSJack Steiner * context. 2359a0deeccSJack Steiner */ 23643884604SJack Steiner static int gru_load_mm_tracker(struct gru_state *gru, 23743884604SJack Steiner struct gru_thread_state *gts) 2389a0deeccSJack Steiner { 23943884604SJack Steiner struct gru_mm_struct *gms = gts->ts_gms; 2409a0deeccSJack Steiner struct gru_mm_tracker *asids = &gms->ms_asids[gru->gs_gid]; 24143884604SJack Steiner unsigned short ctxbitmap = (1 << gts->ts_ctxnum); 2429a0deeccSJack Steiner int asid; 2439a0deeccSJack Steiner 2449a0deeccSJack Steiner spin_lock(&gms->ms_asid_lock); 2459a0deeccSJack Steiner asid = asids->mt_asid; 2469a0deeccSJack Steiner 24787419412SJack Steiner spin_lock(&gru->gs_asid_lock); 24887419412SJack Steiner if (asid == 0 || (asids->mt_ctxbitmap == 0 && asids->mt_asid_gen != 24987419412SJack Steiner gru->gs_asid_gen)) { 2509a0deeccSJack Steiner asid = gru_assign_asid(gru); 2519a0deeccSJack Steiner asids->mt_asid = asid; 2529a0deeccSJack Steiner asids->mt_asid_gen = gru->gs_asid_gen; 2539a0deeccSJack Steiner STAT(asid_new); 2549a0deeccSJack Steiner } else { 2559a0deeccSJack Steiner STAT(asid_reuse); 2569a0deeccSJack Steiner } 25787419412SJack Steiner spin_unlock(&gru->gs_asid_lock); 2589a0deeccSJack Steiner 2599a0deeccSJack Steiner BUG_ON(asids->mt_ctxbitmap & ctxbitmap); 2609a0deeccSJack Steiner asids->mt_ctxbitmap |= ctxbitmap; 2619a0deeccSJack Steiner if (!test_bit(gru->gs_gid, gms->ms_asidmap)) 2629a0deeccSJack Steiner __set_bit(gru->gs_gid, gms->ms_asidmap); 2639a0deeccSJack Steiner spin_unlock(&gms->ms_asid_lock); 2649a0deeccSJack Steiner 2659a0deeccSJack Steiner gru_dbg(grudev, 26643884604SJack Steiner "gid %d, gts %p, gms %p, ctxnum %d, asid 0x%x, asidmap 0x%lx\n", 26743884604SJack Steiner gru->gs_gid, gts, gms, gts->ts_ctxnum, asid, 26843884604SJack Steiner gms->ms_asidmap[0]); 2699a0deeccSJack Steiner return asid; 2709a0deeccSJack Steiner } 2719a0deeccSJack Steiner 2729a0deeccSJack Steiner static void gru_unload_mm_tracker(struct gru_state *gru, 27343884604SJack Steiner struct gru_thread_state *gts) 2749a0deeccSJack Steiner { 27543884604SJack Steiner struct gru_mm_struct *gms = gts->ts_gms; 2769a0deeccSJack Steiner struct gru_mm_tracker *asids; 2779a0deeccSJack Steiner unsigned short ctxbitmap; 2789a0deeccSJack Steiner 2799a0deeccSJack Steiner asids = &gms->ms_asids[gru->gs_gid]; 28043884604SJack Steiner ctxbitmap = (1 << gts->ts_ctxnum); 2819a0deeccSJack Steiner spin_lock(&gms->ms_asid_lock); 28287419412SJack Steiner spin_lock(&gru->gs_asid_lock); 2839a0deeccSJack Steiner BUG_ON((asids->mt_ctxbitmap & ctxbitmap) != ctxbitmap); 2849a0deeccSJack Steiner asids->mt_ctxbitmap ^= ctxbitmap; 28543884604SJack Steiner gru_dbg(grudev, "gid %d, gts %p, gms %p, ctxnum 0x%d, asidmap 0x%lx\n", 28643884604SJack Steiner gru->gs_gid, gts, gms, gts->ts_ctxnum, gms->ms_asidmap[0]); 28787419412SJack Steiner spin_unlock(&gru->gs_asid_lock); 2889a0deeccSJack Steiner spin_unlock(&gms->ms_asid_lock); 2899a0deeccSJack Steiner } 2909a0deeccSJack Steiner 2919a0deeccSJack Steiner /* 2929a0deeccSJack Steiner * Decrement the reference count on a GTS structure. Free the structure 2939a0deeccSJack Steiner * if the reference count goes to zero. 2949a0deeccSJack Steiner */ 2959a0deeccSJack Steiner void gts_drop(struct gru_thread_state *gts) 2969a0deeccSJack Steiner { 2979a0deeccSJack Steiner if (gts && atomic_dec_return(>s->ts_refcnt) == 0) { 2987f2251b1SJack Steiner if (gts->ts_gms) 2999a0deeccSJack Steiner gru_drop_mmu_notifier(gts->ts_gms); 3009a0deeccSJack Steiner kfree(gts); 3019a0deeccSJack Steiner STAT(gts_free); 3029a0deeccSJack Steiner } 3039a0deeccSJack Steiner } 3049a0deeccSJack Steiner 3059a0deeccSJack Steiner /* 3069a0deeccSJack Steiner * Locate the GTS structure for the current thread. 3079a0deeccSJack Steiner */ 3089a0deeccSJack Steiner static struct gru_thread_state *gru_find_current_gts_nolock(struct gru_vma_data 3099a0deeccSJack Steiner *vdata, int tsid) 3109a0deeccSJack Steiner { 3119a0deeccSJack Steiner struct gru_thread_state *gts; 3129a0deeccSJack Steiner 3139a0deeccSJack Steiner list_for_each_entry(gts, &vdata->vd_head, ts_next) 3149a0deeccSJack Steiner if (gts->ts_tsid == tsid) 3159a0deeccSJack Steiner return gts; 3169a0deeccSJack Steiner return NULL; 3179a0deeccSJack Steiner } 3189a0deeccSJack Steiner 3199a0deeccSJack Steiner /* 3209a0deeccSJack Steiner * Allocate a thread state structure. 3219a0deeccSJack Steiner */ 322364b76dfSJack Steiner struct gru_thread_state *gru_alloc_gts(struct vm_area_struct *vma, 323c550222fSJack Steiner int cbr_au_count, int dsr_au_count, 324c550222fSJack Steiner unsigned char tlb_preload_count, int options, int tsid) 3259a0deeccSJack Steiner { 3269a0deeccSJack Steiner struct gru_thread_state *gts; 3277f2251b1SJack Steiner struct gru_mm_struct *gms; 3289a0deeccSJack Steiner int bytes; 3299a0deeccSJack Steiner 330364b76dfSJack Steiner bytes = DSR_BYTES(dsr_au_count) + CBR_BYTES(cbr_au_count); 3319a0deeccSJack Steiner bytes += sizeof(struct gru_thread_state); 332940229b9SJack Steiner gts = kmalloc(bytes, GFP_KERNEL); 3339a0deeccSJack Steiner if (!gts) 3347f2251b1SJack Steiner return ERR_PTR(-ENOMEM); 3359a0deeccSJack Steiner 3369a0deeccSJack Steiner STAT(gts_alloc); 337940229b9SJack Steiner memset(gts, 0, sizeof(struct gru_thread_state)); /* zero out header */ 3389a0deeccSJack Steiner atomic_set(>s->ts_refcnt, 1); 3399a0deeccSJack Steiner mutex_init(>s->ts_ctxlock); 340364b76dfSJack Steiner gts->ts_cbr_au_count = cbr_au_count; 341364b76dfSJack Steiner gts->ts_dsr_au_count = dsr_au_count; 342c550222fSJack Steiner gts->ts_tlb_preload_count = tlb_preload_count; 343364b76dfSJack Steiner gts->ts_user_options = options; 344518e5cd4SJack Steiner gts->ts_user_blade_id = -1; 345518e5cd4SJack Steiner gts->ts_user_chiplet_id = -1; 3469a0deeccSJack Steiner gts->ts_tsid = tsid; 3479a0deeccSJack Steiner gts->ts_ctxnum = NULLCTX; 348364b76dfSJack Steiner gts->ts_tlb_int_select = -1; 349b1b19fcfSJack Steiner gts->ts_cch_req_slice = -1; 350364b76dfSJack Steiner gts->ts_sizeavail = GRU_SIZEAVAIL(PAGE_SHIFT); 351364b76dfSJack Steiner if (vma) { 3529a0deeccSJack Steiner gts->ts_mm = current->mm; 3539a0deeccSJack Steiner gts->ts_vma = vma; 3547f2251b1SJack Steiner gms = gru_register_mmu_notifier(); 3557f2251b1SJack Steiner if (IS_ERR(gms)) 3569a0deeccSJack Steiner goto err; 3577f2251b1SJack Steiner gts->ts_gms = gms; 358364b76dfSJack Steiner } 3599a0deeccSJack Steiner 360364b76dfSJack Steiner gru_dbg(grudev, "alloc gts %p\n", gts); 3619a0deeccSJack Steiner return gts; 3629a0deeccSJack Steiner 3639a0deeccSJack Steiner err: 3649a0deeccSJack Steiner gts_drop(gts); 3657f2251b1SJack Steiner return ERR_CAST(gms); 3669a0deeccSJack Steiner } 3679a0deeccSJack Steiner 3689a0deeccSJack Steiner /* 3699a0deeccSJack Steiner * Allocate a vma private data structure. 3709a0deeccSJack Steiner */ 3719a0deeccSJack Steiner struct gru_vma_data *gru_alloc_vma_data(struct vm_area_struct *vma, int tsid) 3729a0deeccSJack Steiner { 3739a0deeccSJack Steiner struct gru_vma_data *vdata = NULL; 3749a0deeccSJack Steiner 3759a0deeccSJack Steiner vdata = kmalloc(sizeof(*vdata), GFP_KERNEL); 3769a0deeccSJack Steiner if (!vdata) 3779a0deeccSJack Steiner return NULL; 3789a0deeccSJack Steiner 379563447d7SJack Steiner STAT(vdata_alloc); 3809a0deeccSJack Steiner INIT_LIST_HEAD(&vdata->vd_head); 3819a0deeccSJack Steiner spin_lock_init(&vdata->vd_lock); 3829a0deeccSJack Steiner gru_dbg(grudev, "alloc vdata %p\n", vdata); 3839a0deeccSJack Steiner return vdata; 3849a0deeccSJack Steiner } 3859a0deeccSJack Steiner 3869a0deeccSJack Steiner /* 3879a0deeccSJack Steiner * Find the thread state structure for the current thread. 3889a0deeccSJack Steiner */ 3899a0deeccSJack Steiner struct gru_thread_state *gru_find_thread_state(struct vm_area_struct *vma, 3909a0deeccSJack Steiner int tsid) 3919a0deeccSJack Steiner { 3929a0deeccSJack Steiner struct gru_vma_data *vdata = vma->vm_private_data; 3939a0deeccSJack Steiner struct gru_thread_state *gts; 3949a0deeccSJack Steiner 3959a0deeccSJack Steiner spin_lock(&vdata->vd_lock); 3969a0deeccSJack Steiner gts = gru_find_current_gts_nolock(vdata, tsid); 3979a0deeccSJack Steiner spin_unlock(&vdata->vd_lock); 3989a0deeccSJack Steiner gru_dbg(grudev, "vma %p, gts %p\n", vma, gts); 3999a0deeccSJack Steiner return gts; 4009a0deeccSJack Steiner } 4019a0deeccSJack Steiner 4029a0deeccSJack Steiner /* 4039a0deeccSJack Steiner * Allocate a new thread state for a GSEG. Note that races may allow 4049a0deeccSJack Steiner * another thread to race to create a gts. 4059a0deeccSJack Steiner */ 4069a0deeccSJack Steiner struct gru_thread_state *gru_alloc_thread_state(struct vm_area_struct *vma, 4079a0deeccSJack Steiner int tsid) 4089a0deeccSJack Steiner { 4099a0deeccSJack Steiner struct gru_vma_data *vdata = vma->vm_private_data; 4109a0deeccSJack Steiner struct gru_thread_state *gts, *ngts; 4119a0deeccSJack Steiner 412c550222fSJack Steiner gts = gru_alloc_gts(vma, vdata->vd_cbr_au_count, 413c550222fSJack Steiner vdata->vd_dsr_au_count, 414c550222fSJack Steiner vdata->vd_tlb_preload_count, 415364b76dfSJack Steiner vdata->vd_user_options, tsid); 4167f2251b1SJack Steiner if (IS_ERR(gts)) 4177f2251b1SJack Steiner return gts; 4189a0deeccSJack Steiner 4199a0deeccSJack Steiner spin_lock(&vdata->vd_lock); 4209a0deeccSJack Steiner ngts = gru_find_current_gts_nolock(vdata, tsid); 4219a0deeccSJack Steiner if (ngts) { 4229a0deeccSJack Steiner gts_drop(gts); 4239a0deeccSJack Steiner gts = ngts; 4249a0deeccSJack Steiner STAT(gts_double_allocate); 4259a0deeccSJack Steiner } else { 4269a0deeccSJack Steiner list_add(>s->ts_next, &vdata->vd_head); 4279a0deeccSJack Steiner } 4289a0deeccSJack Steiner spin_unlock(&vdata->vd_lock); 4299a0deeccSJack Steiner gru_dbg(grudev, "vma %p, gts %p\n", vma, gts); 4309a0deeccSJack Steiner return gts; 4319a0deeccSJack Steiner } 4329a0deeccSJack Steiner 4339a0deeccSJack Steiner /* 4349a0deeccSJack Steiner * Free the GRU context assigned to the thread state. 4359a0deeccSJack Steiner */ 4369a0deeccSJack Steiner static void gru_free_gru_context(struct gru_thread_state *gts) 4379a0deeccSJack Steiner { 4389a0deeccSJack Steiner struct gru_state *gru; 4399a0deeccSJack Steiner 4409a0deeccSJack Steiner gru = gts->ts_gru; 44143884604SJack Steiner gru_dbg(grudev, "gts %p, gid %d\n", gts, gru->gs_gid); 4429a0deeccSJack Steiner 4439a0deeccSJack Steiner spin_lock(&gru->gs_lock); 4449a0deeccSJack Steiner gru->gs_gts[gts->ts_ctxnum] = NULL; 4459a0deeccSJack Steiner free_gru_resources(gru, gts); 4469a0deeccSJack Steiner BUG_ON(test_bit(gts->ts_ctxnum, &gru->gs_context_map) == 0); 4479a0deeccSJack Steiner __clear_bit(gts->ts_ctxnum, &gru->gs_context_map); 4489a0deeccSJack Steiner gts->ts_ctxnum = NULLCTX; 4499a0deeccSJack Steiner gts->ts_gru = NULL; 45087419412SJack Steiner gts->ts_blade = -1; 4519a0deeccSJack Steiner spin_unlock(&gru->gs_lock); 4529a0deeccSJack Steiner 4539a0deeccSJack Steiner gts_drop(gts); 4549a0deeccSJack Steiner STAT(free_context); 4559a0deeccSJack Steiner } 4569a0deeccSJack Steiner 4579a0deeccSJack Steiner /* 4589a0deeccSJack Steiner * Prefetching cachelines help hardware performance. 4599ca8e40cSJack Steiner * (Strictly a performance enhancement. Not functionally required). 4609a0deeccSJack Steiner */ 4619a0deeccSJack Steiner static void prefetch_data(void *p, int num, int stride) 4629a0deeccSJack Steiner { 4639a0deeccSJack Steiner while (num-- > 0) { 4649a0deeccSJack Steiner prefetchw(p); 4659a0deeccSJack Steiner p += stride; 4669a0deeccSJack Steiner } 4679a0deeccSJack Steiner } 4689a0deeccSJack Steiner 4699a0deeccSJack Steiner static inline long gru_copy_handle(void *d, void *s) 4709a0deeccSJack Steiner { 4719a0deeccSJack Steiner memcpy(d, s, GRU_HANDLE_BYTES); 4729a0deeccSJack Steiner return GRU_HANDLE_BYTES; 4739a0deeccSJack Steiner } 4749a0deeccSJack Steiner 475fe5bb6b0SJack Steiner static void gru_prefetch_context(void *gseg, void *cb, void *cbe, 476fe5bb6b0SJack Steiner unsigned long cbrmap, unsigned long length) 477923f7f69SJack Steiner { 478923f7f69SJack Steiner int i, scr; 479923f7f69SJack Steiner 480923f7f69SJack Steiner prefetch_data(gseg + GRU_DS_BASE, length / GRU_CACHE_LINE_BYTES, 481923f7f69SJack Steiner GRU_CACHE_LINE_BYTES); 482923f7f69SJack Steiner 483923f7f69SJack Steiner for_each_cbr_in_allocation_map(i, &cbrmap, scr) { 484923f7f69SJack Steiner prefetch_data(cb, 1, GRU_CACHE_LINE_BYTES); 485923f7f69SJack Steiner prefetch_data(cbe + i * GRU_HANDLE_STRIDE, 1, 486923f7f69SJack Steiner GRU_CACHE_LINE_BYTES); 487923f7f69SJack Steiner cb += GRU_HANDLE_STRIDE; 488923f7f69SJack Steiner } 489923f7f69SJack Steiner } 490923f7f69SJack Steiner 4919a0deeccSJack Steiner static void gru_load_context_data(void *save, void *grubase, int ctxnum, 492940229b9SJack Steiner unsigned long cbrmap, unsigned long dsrmap, 493940229b9SJack Steiner int data_valid) 4949a0deeccSJack Steiner { 4959a0deeccSJack Steiner void *gseg, *cb, *cbe; 4969a0deeccSJack Steiner unsigned long length; 4979a0deeccSJack Steiner int i, scr; 4989a0deeccSJack Steiner 4999a0deeccSJack Steiner gseg = grubase + ctxnum * GRU_GSEG_STRIDE; 5009a0deeccSJack Steiner cb = gseg + GRU_CB_BASE; 5019a0deeccSJack Steiner cbe = grubase + GRU_CBE_BASE; 502923f7f69SJack Steiner length = hweight64(dsrmap) * GRU_DSR_AU_BYTES; 503923f7f69SJack Steiner gru_prefetch_context(gseg, cb, cbe, cbrmap, length); 5049a0deeccSJack Steiner 5059a0deeccSJack Steiner for_each_cbr_in_allocation_map(i, &cbrmap, scr) { 506940229b9SJack Steiner if (data_valid) { 5079a0deeccSJack Steiner save += gru_copy_handle(cb, save); 508940229b9SJack Steiner save += gru_copy_handle(cbe + i * GRU_HANDLE_STRIDE, 509940229b9SJack Steiner save); 510940229b9SJack Steiner } else { 511940229b9SJack Steiner memset(cb, 0, GRU_CACHE_LINE_BYTES); 512940229b9SJack Steiner memset(cbe + i * GRU_HANDLE_STRIDE, 0, 513940229b9SJack Steiner GRU_CACHE_LINE_BYTES); 514940229b9SJack Steiner } 51567bf04a5SJack Steiner /* Flush CBE to hide race in context restart */ 51667bf04a5SJack Steiner mb(); 51767bf04a5SJack Steiner gru_flush_cache(cbe + i * GRU_HANDLE_STRIDE); 5189a0deeccSJack Steiner cb += GRU_HANDLE_STRIDE; 5199a0deeccSJack Steiner } 5209a0deeccSJack Steiner 521940229b9SJack Steiner if (data_valid) 5229a0deeccSJack Steiner memcpy(gseg + GRU_DS_BASE, save, length); 523940229b9SJack Steiner else 524940229b9SJack Steiner memset(gseg + GRU_DS_BASE, 0, length); 5259a0deeccSJack Steiner } 5269a0deeccSJack Steiner 5279a0deeccSJack Steiner static void gru_unload_context_data(void *save, void *grubase, int ctxnum, 5289a0deeccSJack Steiner unsigned long cbrmap, unsigned long dsrmap) 5299a0deeccSJack Steiner { 5309a0deeccSJack Steiner void *gseg, *cb, *cbe; 5319a0deeccSJack Steiner unsigned long length; 5329a0deeccSJack Steiner int i, scr; 5339a0deeccSJack Steiner 5349a0deeccSJack Steiner gseg = grubase + ctxnum * GRU_GSEG_STRIDE; 5359a0deeccSJack Steiner cb = gseg + GRU_CB_BASE; 5369a0deeccSJack Steiner cbe = grubase + GRU_CBE_BASE; 537923f7f69SJack Steiner length = hweight64(dsrmap) * GRU_DSR_AU_BYTES; 53867bf04a5SJack Steiner 53967bf04a5SJack Steiner /* CBEs may not be coherent. Flush them from cache */ 54067bf04a5SJack Steiner for_each_cbr_in_allocation_map(i, &cbrmap, scr) 54167bf04a5SJack Steiner gru_flush_cache(cbe + i * GRU_HANDLE_STRIDE); 54267bf04a5SJack Steiner mb(); /* Let the CL flush complete */ 54367bf04a5SJack Steiner 544923f7f69SJack Steiner gru_prefetch_context(gseg, cb, cbe, cbrmap, length); 545923f7f69SJack Steiner 5469a0deeccSJack Steiner for_each_cbr_in_allocation_map(i, &cbrmap, scr) { 5479a0deeccSJack Steiner save += gru_copy_handle(save, cb); 5489a0deeccSJack Steiner save += gru_copy_handle(save, cbe + i * GRU_HANDLE_STRIDE); 5499a0deeccSJack Steiner cb += GRU_HANDLE_STRIDE; 5509a0deeccSJack Steiner } 5519a0deeccSJack Steiner memcpy(save, gseg + GRU_DS_BASE, length); 5529a0deeccSJack Steiner } 5539a0deeccSJack Steiner 5549a0deeccSJack Steiner void gru_unload_context(struct gru_thread_state *gts, int savestate) 5559a0deeccSJack Steiner { 5569a0deeccSJack Steiner struct gru_state *gru = gts->ts_gru; 5579a0deeccSJack Steiner struct gru_context_configuration_handle *cch; 5589a0deeccSJack Steiner int ctxnum = gts->ts_ctxnum; 5599a0deeccSJack Steiner 560836ce679SJack Steiner if (!is_kernel_context(gts)) 5619a0deeccSJack Steiner zap_vma_ptes(gts->ts_vma, UGRUADDR(gts), GRU_GSEG_PAGESIZE); 5629a0deeccSJack Steiner cch = get_cch(gru->gs_gru_base_vaddr, ctxnum); 5639a0deeccSJack Steiner 564563447d7SJack Steiner gru_dbg(grudev, "gts %p, cbrmap 0x%lx, dsrmap 0x%lx\n", 565563447d7SJack Steiner gts, gts->ts_cbr_map, gts->ts_dsr_map); 5669a0deeccSJack Steiner lock_cch_handle(cch); 5679a0deeccSJack Steiner if (cch_interrupt_sync(cch)) 5689a0deeccSJack Steiner BUG(); 5699a0deeccSJack Steiner 570836ce679SJack Steiner if (!is_kernel_context(gts)) 57143884604SJack Steiner gru_unload_mm_tracker(gru, gts); 572940229b9SJack Steiner if (savestate) { 5739a0deeccSJack Steiner gru_unload_context_data(gts->ts_gdata, gru->gs_gru_base_vaddr, 5749a0deeccSJack Steiner ctxnum, gts->ts_cbr_map, 5759a0deeccSJack Steiner gts->ts_dsr_map); 576940229b9SJack Steiner gts->ts_data_valid = 1; 577940229b9SJack Steiner } 5789a0deeccSJack Steiner 5799a0deeccSJack Steiner if (cch_deallocate(cch)) 5809a0deeccSJack Steiner BUG(); 5819a0deeccSJack Steiner unlock_cch_handle(cch); 5829a0deeccSJack Steiner 5839a0deeccSJack Steiner gru_free_gru_context(gts); 5849a0deeccSJack Steiner } 5859a0deeccSJack Steiner 5869a0deeccSJack Steiner /* 5879a0deeccSJack Steiner * Load a GRU context by copying it from the thread data structure in memory 5889a0deeccSJack Steiner * to the GRU. 5899a0deeccSJack Steiner */ 590d57c82b1SJack Steiner void gru_load_context(struct gru_thread_state *gts) 5919a0deeccSJack Steiner { 5929a0deeccSJack Steiner struct gru_state *gru = gts->ts_gru; 5939a0deeccSJack Steiner struct gru_context_configuration_handle *cch; 5946e910074SJack Steiner int i, err, asid, ctxnum = gts->ts_ctxnum; 5959a0deeccSJack Steiner 5969a0deeccSJack Steiner cch = get_cch(gru->gs_gru_base_vaddr, ctxnum); 5979a0deeccSJack Steiner lock_cch_handle(cch); 5989a0deeccSJack Steiner cch->tfm_fault_bit_enable = 5999a0deeccSJack Steiner (gts->ts_user_options == GRU_OPT_MISS_FMM_POLL 6009a0deeccSJack Steiner || gts->ts_user_options == GRU_OPT_MISS_FMM_INTR); 6019a0deeccSJack Steiner cch->tlb_int_enable = (gts->ts_user_options == GRU_OPT_MISS_FMM_INTR); 6029a0deeccSJack Steiner if (cch->tlb_int_enable) { 6039a0deeccSJack Steiner gts->ts_tlb_int_select = gru_cpu_fault_map_id(); 6049a0deeccSJack Steiner cch->tlb_int_select = gts->ts_tlb_int_select; 6059a0deeccSJack Steiner } 606b1b19fcfSJack Steiner if (gts->ts_cch_req_slice >= 0) { 607b1b19fcfSJack Steiner cch->req_slice_set_enable = 1; 608b1b19fcfSJack Steiner cch->req_slice = gts->ts_cch_req_slice; 609b1b19fcfSJack Steiner } else { 610b1b19fcfSJack Steiner cch->req_slice_set_enable =0; 611b1b19fcfSJack Steiner } 6129a0deeccSJack Steiner cch->tfm_done_bit_enable = 0; 6136e910074SJack Steiner cch->dsr_allocation_map = gts->ts_dsr_map; 6146e910074SJack Steiner cch->cbr_allocation_map = gts->ts_cbr_map; 615836ce679SJack Steiner 616836ce679SJack Steiner if (is_kernel_context(gts)) { 617836ce679SJack Steiner cch->unmap_enable = 1; 6184a7a17c1SJack Steiner cch->tfm_done_bit_enable = 1; 6194a7a17c1SJack Steiner cch->cb_int_enable = 1; 6204107e1d3SJack Steiner cch->tlb_int_select = 0; /* For now, ints go to cpu 0 */ 621836ce679SJack Steiner } else { 6226e910074SJack Steiner cch->unmap_enable = 0; 6234a7a17c1SJack Steiner cch->tfm_done_bit_enable = 0; 6244a7a17c1SJack Steiner cch->cb_int_enable = 0; 625836ce679SJack Steiner asid = gru_load_mm_tracker(gru, gts); 6266e910074SJack Steiner for (i = 0; i < 8; i++) { 6276e910074SJack Steiner cch->asid[i] = asid + i; 6286e910074SJack Steiner cch->sizeavail[i] = gts->ts_sizeavail; 6296e910074SJack Steiner } 630836ce679SJack Steiner } 6316e910074SJack Steiner 6326e910074SJack Steiner err = cch_allocate(cch); 6339a0deeccSJack Steiner if (err) { 6349a0deeccSJack Steiner gru_dbg(grudev, 6359a0deeccSJack Steiner "err %d: cch %p, gts %p, cbr 0x%lx, dsr 0x%lx\n", 6369a0deeccSJack Steiner err, cch, gts, gts->ts_cbr_map, gts->ts_dsr_map); 6379a0deeccSJack Steiner BUG(); 6389a0deeccSJack Steiner } 6399a0deeccSJack Steiner 6409a0deeccSJack Steiner gru_load_context_data(gts->ts_gdata, gru->gs_gru_base_vaddr, ctxnum, 641940229b9SJack Steiner gts->ts_cbr_map, gts->ts_dsr_map, gts->ts_data_valid); 6429a0deeccSJack Steiner 6439a0deeccSJack Steiner if (cch_start(cch)) 6449a0deeccSJack Steiner BUG(); 6459a0deeccSJack Steiner unlock_cch_handle(cch); 646563447d7SJack Steiner 647563447d7SJack Steiner gru_dbg(grudev, "gid %d, gts %p, cbrmap 0x%lx, dsrmap 0x%lx, tie %d, tis %d\n", 648563447d7SJack Steiner gts->ts_gru->gs_gid, gts, gts->ts_cbr_map, gts->ts_dsr_map, 649563447d7SJack Steiner (gts->ts_user_options == GRU_OPT_MISS_FMM_INTR), gts->ts_tlb_int_select); 6509a0deeccSJack Steiner } 6519a0deeccSJack Steiner 6529a0deeccSJack Steiner /* 6539a0deeccSJack Steiner * Update fields in an active CCH: 6549a0deeccSJack Steiner * - retarget interrupts on local blade 6557b8274e9SJack Steiner * - update sizeavail mask 6569a0deeccSJack Steiner */ 65799f7c229SJack Steiner int gru_update_cch(struct gru_thread_state *gts) 6589a0deeccSJack Steiner { 6599a0deeccSJack Steiner struct gru_context_configuration_handle *cch; 6609a0deeccSJack Steiner struct gru_state *gru = gts->ts_gru; 6619a0deeccSJack Steiner int i, ctxnum = gts->ts_ctxnum, ret = 0; 6629a0deeccSJack Steiner 6639a0deeccSJack Steiner cch = get_cch(gru->gs_gru_base_vaddr, ctxnum); 6649a0deeccSJack Steiner 6659a0deeccSJack Steiner lock_cch_handle(cch); 6669a0deeccSJack Steiner if (cch->state == CCHSTATE_ACTIVE) { 6679a0deeccSJack Steiner if (gru->gs_gts[gts->ts_ctxnum] != gts) 6689a0deeccSJack Steiner goto exit; 6699a0deeccSJack Steiner if (cch_interrupt(cch)) 6709a0deeccSJack Steiner BUG(); 6717b8274e9SJack Steiner for (i = 0; i < 8; i++) 6727b8274e9SJack Steiner cch->sizeavail[i] = gts->ts_sizeavail; 6737b8274e9SJack Steiner gts->ts_tlb_int_select = gru_cpu_fault_map_id(); 6747b8274e9SJack Steiner cch->tlb_int_select = gru_cpu_fault_map_id(); 675cd1334f0SJack Steiner cch->tfm_fault_bit_enable = 676cd1334f0SJack Steiner (gts->ts_user_options == GRU_OPT_MISS_FMM_POLL 677cd1334f0SJack Steiner || gts->ts_user_options == GRU_OPT_MISS_FMM_INTR); 6789a0deeccSJack Steiner if (cch_start(cch)) 6799a0deeccSJack Steiner BUG(); 6809a0deeccSJack Steiner ret = 1; 6819a0deeccSJack Steiner } 6829a0deeccSJack Steiner exit: 6839a0deeccSJack Steiner unlock_cch_handle(cch); 6849a0deeccSJack Steiner return ret; 6859a0deeccSJack Steiner } 6869a0deeccSJack Steiner 6879a0deeccSJack Steiner /* 6889a0deeccSJack Steiner * Update CCH tlb interrupt select. Required when all the following is true: 6899a0deeccSJack Steiner * - task's GRU context is loaded into a GRU 6909a0deeccSJack Steiner * - task is using interrupt notification for TLB faults 6919a0deeccSJack Steiner * - task has migrated to a different cpu on the same blade where 6929a0deeccSJack Steiner * it was previously running. 6939a0deeccSJack Steiner */ 6949a0deeccSJack Steiner static int gru_retarget_intr(struct gru_thread_state *gts) 6959a0deeccSJack Steiner { 6969a0deeccSJack Steiner if (gts->ts_tlb_int_select < 0 6979a0deeccSJack Steiner || gts->ts_tlb_int_select == gru_cpu_fault_map_id()) 6989a0deeccSJack Steiner return 0; 6999a0deeccSJack Steiner 7009a0deeccSJack Steiner gru_dbg(grudev, "retarget from %d to %d\n", gts->ts_tlb_int_select, 7019a0deeccSJack Steiner gru_cpu_fault_map_id()); 70299f7c229SJack Steiner return gru_update_cch(gts); 7039a0deeccSJack Steiner } 7049a0deeccSJack Steiner 70555484c45SJack Steiner /* 70641314790SJack Steiner * Check if a GRU context is allowed to use a specific chiplet. By default 70741314790SJack Steiner * a context is assigned to any blade-local chiplet. However, users can 70841314790SJack Steiner * override this. 70941314790SJack Steiner * Returns 1 if assignment allowed, 0 otherwise 71041314790SJack Steiner */ 71141314790SJack Steiner static int gru_check_chiplet_assignment(struct gru_state *gru, 71241314790SJack Steiner struct gru_thread_state *gts) 71341314790SJack Steiner { 71441314790SJack Steiner int blade_id; 71541314790SJack Steiner int chiplet_id; 71641314790SJack Steiner 71741314790SJack Steiner blade_id = gts->ts_user_blade_id; 71841314790SJack Steiner if (blade_id < 0) 71941314790SJack Steiner blade_id = uv_numa_blade_id(); 72041314790SJack Steiner 72141314790SJack Steiner chiplet_id = gts->ts_user_chiplet_id; 72241314790SJack Steiner return gru->gs_blade_id == blade_id && 72341314790SJack Steiner (chiplet_id < 0 || chiplet_id == gru->gs_chiplet_id); 72441314790SJack Steiner } 72541314790SJack Steiner 72641314790SJack Steiner /* 72755484c45SJack Steiner * Unload the gru context if it is not assigned to the correct blade or 72855484c45SJack Steiner * chiplet. Misassignment can occur if the process migrates to a different 72955484c45SJack Steiner * blade or if the user changes the selected blade/chiplet. 73055484c45SJack Steiner */ 73155484c45SJack Steiner void gru_check_context_placement(struct gru_thread_state *gts) 73255484c45SJack Steiner { 73355484c45SJack Steiner struct gru_state *gru; 73455484c45SJack Steiner 73555484c45SJack Steiner /* 73655484c45SJack Steiner * If the current task is the context owner, verify that the 73755484c45SJack Steiner * context is correctly placed. This test is skipped for non-owner 73855484c45SJack Steiner * references. Pthread apps use non-owner references to the CBRs. 73955484c45SJack Steiner */ 74055484c45SJack Steiner gru = gts->ts_gru; 74155484c45SJack Steiner if (!gru || gts->ts_tgid_owner != current->tgid) 74255484c45SJack Steiner return; 74355484c45SJack Steiner 74441314790SJack Steiner if (!gru_check_chiplet_assignment(gru, gts)) { 74555484c45SJack Steiner STAT(check_context_unload); 74655484c45SJack Steiner gru_unload_context(gts, 1); 74755484c45SJack Steiner } else if (gru_retarget_intr(gts)) { 74855484c45SJack Steiner STAT(check_context_retarget_intr); 74955484c45SJack Steiner } 75055484c45SJack Steiner } 75155484c45SJack Steiner 7529a0deeccSJack Steiner 7539a0deeccSJack Steiner /* 7549a0deeccSJack Steiner * Insufficient GRU resources available on the local blade. Steal a context from 7559a0deeccSJack Steiner * a process. This is a hack until a _real_ resource scheduler is written.... 7569a0deeccSJack Steiner */ 7579a0deeccSJack Steiner #define next_ctxnum(n) ((n) < GRU_NUM_CCH - 2 ? (n) + 1 : 0) 7589a0deeccSJack Steiner #define next_gru(b, g) (((g) < &(b)->bs_grus[GRU_CHIPLETS_PER_BLADE - 1]) ? \ 7599a0deeccSJack Steiner ((g)+1) : &(b)->bs_grus[0]) 7609a0deeccSJack Steiner 761836ce679SJack Steiner static int is_gts_stealable(struct gru_thread_state *gts, 762836ce679SJack Steiner struct gru_blade_state *bs) 763836ce679SJack Steiner { 764836ce679SJack Steiner if (is_kernel_context(gts)) 765836ce679SJack Steiner return down_write_trylock(&bs->bs_kgts_sema); 766836ce679SJack Steiner else 767836ce679SJack Steiner return mutex_trylock(>s->ts_ctxlock); 768836ce679SJack Steiner } 769836ce679SJack Steiner 770836ce679SJack Steiner static void gts_stolen(struct gru_thread_state *gts, 771836ce679SJack Steiner struct gru_blade_state *bs) 772836ce679SJack Steiner { 773836ce679SJack Steiner if (is_kernel_context(gts)) { 774836ce679SJack Steiner up_write(&bs->bs_kgts_sema); 775836ce679SJack Steiner STAT(steal_kernel_context); 776836ce679SJack Steiner } else { 777836ce679SJack Steiner mutex_unlock(>s->ts_ctxlock); 778836ce679SJack Steiner STAT(steal_user_context); 779836ce679SJack Steiner } 780836ce679SJack Steiner } 781836ce679SJack Steiner 78255484c45SJack Steiner void gru_steal_context(struct gru_thread_state *gts) 7839a0deeccSJack Steiner { 7849a0deeccSJack Steiner struct gru_blade_state *blade; 7859a0deeccSJack Steiner struct gru_state *gru, *gru0; 7869a0deeccSJack Steiner struct gru_thread_state *ngts = NULL; 7879a0deeccSJack Steiner int ctxnum, ctxnum0, flag = 0, cbr, dsr; 78841314790SJack Steiner int blade_id; 7899a0deeccSJack Steiner 79041314790SJack Steiner blade_id = gts->ts_user_blade_id; 79155484c45SJack Steiner if (blade_id < 0) 79255484c45SJack Steiner blade_id = uv_numa_blade_id(); 7939a0deeccSJack Steiner cbr = gts->ts_cbr_au_count; 7949a0deeccSJack Steiner dsr = gts->ts_dsr_au_count; 7959a0deeccSJack Steiner 796364b76dfSJack Steiner blade = gru_base[blade_id]; 7979a0deeccSJack Steiner spin_lock(&blade->bs_lock); 7989a0deeccSJack Steiner 7999a0deeccSJack Steiner ctxnum = next_ctxnum(blade->bs_lru_ctxnum); 8009a0deeccSJack Steiner gru = blade->bs_lru_gru; 8019a0deeccSJack Steiner if (ctxnum == 0) 8029a0deeccSJack Steiner gru = next_gru(blade, gru); 80355484c45SJack Steiner blade->bs_lru_gru = gru; 80455484c45SJack Steiner blade->bs_lru_ctxnum = ctxnum; 8059a0deeccSJack Steiner ctxnum0 = ctxnum; 8069a0deeccSJack Steiner gru0 = gru; 8079a0deeccSJack Steiner while (1) { 80841314790SJack Steiner if (gru_check_chiplet_assignment(gru, gts)) { 8099a0deeccSJack Steiner if (check_gru_resources(gru, cbr, dsr, GRU_NUM_CCH)) 8109a0deeccSJack Steiner break; 8119a0deeccSJack Steiner spin_lock(&gru->gs_lock); 8129a0deeccSJack Steiner for (; ctxnum < GRU_NUM_CCH; ctxnum++) { 8139a0deeccSJack Steiner if (flag && gru == gru0 && ctxnum == ctxnum0) 8149a0deeccSJack Steiner break; 8159a0deeccSJack Steiner ngts = gru->gs_gts[ctxnum]; 8169a0deeccSJack Steiner /* 8179a0deeccSJack Steiner * We are grabbing locks out of order, so trylock is 8189a0deeccSJack Steiner * needed. GTSs are usually not locked, so the odds of 8199a0deeccSJack Steiner * success are high. If trylock fails, try to steal a 8209a0deeccSJack Steiner * different GSEG. 8219a0deeccSJack Steiner */ 822836ce679SJack Steiner if (ngts && is_gts_stealable(ngts, blade)) 8239a0deeccSJack Steiner break; 8249a0deeccSJack Steiner ngts = NULL; 8259a0deeccSJack Steiner } 8269a0deeccSJack Steiner spin_unlock(&gru->gs_lock); 8279a0deeccSJack Steiner if (ngts || (flag && gru == gru0 && ctxnum == ctxnum0)) 8289a0deeccSJack Steiner break; 82955484c45SJack Steiner } 83055484c45SJack Steiner if (flag && gru == gru0) 83155484c45SJack Steiner break; 83255484c45SJack Steiner flag = 1; 8339a0deeccSJack Steiner ctxnum = 0; 8349a0deeccSJack Steiner gru = next_gru(blade, gru); 8359a0deeccSJack Steiner } 8369a0deeccSJack Steiner spin_unlock(&blade->bs_lock); 8379a0deeccSJack Steiner 8389a0deeccSJack Steiner if (ngts) { 8397e796a72SJack Steiner gts->ustats.context_stolen++; 8409a0deeccSJack Steiner ngts->ts_steal_jiffies = jiffies; 841836ce679SJack Steiner gru_unload_context(ngts, is_kernel_context(ngts) ? 0 : 1); 842836ce679SJack Steiner gts_stolen(ngts, blade); 8439a0deeccSJack Steiner } else { 8449a0deeccSJack Steiner STAT(steal_context_failed); 8459a0deeccSJack Steiner } 8469a0deeccSJack Steiner gru_dbg(grudev, 84743884604SJack Steiner "stole gid %d, ctxnum %d from gts %p. Need cb %d, ds %d;" 8489a0deeccSJack Steiner " avail cb %ld, ds %ld\n", 8499a0deeccSJack Steiner gru->gs_gid, ctxnum, ngts, cbr, dsr, hweight64(gru->gs_cbr_map), 8509a0deeccSJack Steiner hweight64(gru->gs_dsr_map)); 8519a0deeccSJack Steiner } 8529a0deeccSJack Steiner 8539a0deeccSJack Steiner /* 85455484c45SJack Steiner * Assign a gru context. 85555484c45SJack Steiner */ 85655484c45SJack Steiner static int gru_assign_context_number(struct gru_state *gru) 85755484c45SJack Steiner { 85855484c45SJack Steiner int ctxnum; 85955484c45SJack Steiner 86055484c45SJack Steiner ctxnum = find_first_zero_bit(&gru->gs_context_map, GRU_NUM_CCH); 86155484c45SJack Steiner __set_bit(ctxnum, &gru->gs_context_map); 86255484c45SJack Steiner return ctxnum; 86355484c45SJack Steiner } 86455484c45SJack Steiner 86555484c45SJack Steiner /* 8669a0deeccSJack Steiner * Scan the GRUs on the local blade & assign a GRU context. 8679a0deeccSJack Steiner */ 86855484c45SJack Steiner struct gru_state *gru_assign_gru_context(struct gru_thread_state *gts) 8699a0deeccSJack Steiner { 8709a0deeccSJack Steiner struct gru_state *gru, *grux; 8719a0deeccSJack Steiner int i, max_active_contexts; 87255484c45SJack Steiner int blade_id = gts->ts_user_blade_id; 8739a0deeccSJack Steiner 87455484c45SJack Steiner if (blade_id < 0) 87555484c45SJack Steiner blade_id = uv_numa_blade_id(); 8769a0deeccSJack Steiner again: 8779a0deeccSJack Steiner gru = NULL; 8789a0deeccSJack Steiner max_active_contexts = GRU_NUM_CCH; 87955484c45SJack Steiner for_each_gru_on_blade(grux, blade_id, i) { 88041314790SJack Steiner if (!gru_check_chiplet_assignment(grux, gts)) 88155484c45SJack Steiner continue; 8829a0deeccSJack Steiner if (check_gru_resources(grux, gts->ts_cbr_au_count, 8839a0deeccSJack Steiner gts->ts_dsr_au_count, 8849a0deeccSJack Steiner max_active_contexts)) { 8859a0deeccSJack Steiner gru = grux; 8869a0deeccSJack Steiner max_active_contexts = grux->gs_active_contexts; 8879a0deeccSJack Steiner if (max_active_contexts == 0) 8889a0deeccSJack Steiner break; 8899a0deeccSJack Steiner } 8909a0deeccSJack Steiner } 8919a0deeccSJack Steiner 8929a0deeccSJack Steiner if (gru) { 8939a0deeccSJack Steiner spin_lock(&gru->gs_lock); 8949a0deeccSJack Steiner if (!check_gru_resources(gru, gts->ts_cbr_au_count, 8959a0deeccSJack Steiner gts->ts_dsr_au_count, GRU_NUM_CCH)) { 8969a0deeccSJack Steiner spin_unlock(&gru->gs_lock); 8979a0deeccSJack Steiner goto again; 8989a0deeccSJack Steiner } 8999a0deeccSJack Steiner reserve_gru_resources(gru, gts); 9009a0deeccSJack Steiner gts->ts_gru = gru; 90187419412SJack Steiner gts->ts_blade = gru->gs_blade_id; 90255484c45SJack Steiner gts->ts_ctxnum = gru_assign_context_number(gru); 9039a0deeccSJack Steiner atomic_inc(>s->ts_refcnt); 9049a0deeccSJack Steiner gru->gs_gts[gts->ts_ctxnum] = gts; 9059a0deeccSJack Steiner spin_unlock(&gru->gs_lock); 9069a0deeccSJack Steiner 9079a0deeccSJack Steiner STAT(assign_context); 9089a0deeccSJack Steiner gru_dbg(grudev, 90943884604SJack Steiner "gseg %p, gts %p, gid %d, ctx %d, cbr %d, dsr %d\n", 9109a0deeccSJack Steiner gseg_virtual_address(gts->ts_gru, gts->ts_ctxnum), gts, 9119a0deeccSJack Steiner gts->ts_gru->gs_gid, gts->ts_ctxnum, 9129a0deeccSJack Steiner gts->ts_cbr_au_count, gts->ts_dsr_au_count); 9139a0deeccSJack Steiner } else { 9149a0deeccSJack Steiner gru_dbg(grudev, "failed to allocate a GTS %s\n", ""); 9159a0deeccSJack Steiner STAT(assign_context_failed); 9169a0deeccSJack Steiner } 9179a0deeccSJack Steiner 9189a0deeccSJack Steiner return gru; 9199a0deeccSJack Steiner } 9209a0deeccSJack Steiner 9219a0deeccSJack Steiner /* 9229a0deeccSJack Steiner * gru_nopage 9239a0deeccSJack Steiner * 9249a0deeccSJack Steiner * Map the user's GRU segment 9259ca8e40cSJack Steiner * 9269ca8e40cSJack Steiner * Note: gru segments alway mmaped on GRU_GSEG_PAGESIZE boundaries. 9279a0deeccSJack Steiner */ 9289a0deeccSJack Steiner int gru_fault(struct vm_area_struct *vma, struct vm_fault *vmf) 9299a0deeccSJack Steiner { 9309a0deeccSJack Steiner struct gru_thread_state *gts; 9319a0deeccSJack Steiner unsigned long paddr, vaddr; 9329a0deeccSJack Steiner 9339a0deeccSJack Steiner vaddr = (unsigned long)vmf->virtual_address; 9349a0deeccSJack Steiner gru_dbg(grudev, "vma %p, vaddr 0x%lx (0x%lx)\n", 9359a0deeccSJack Steiner vma, vaddr, GSEG_BASE(vaddr)); 9369a0deeccSJack Steiner STAT(nopfn); 9379a0deeccSJack Steiner 9389ca8e40cSJack Steiner /* The following check ensures vaddr is a valid address in the VMA */ 9399a0deeccSJack Steiner gts = gru_find_thread_state(vma, TSID(vaddr, vma)); 9409a0deeccSJack Steiner if (!gts) 9419a0deeccSJack Steiner return VM_FAULT_SIGBUS; 9429a0deeccSJack Steiner 9439a0deeccSJack Steiner again: 9449a0deeccSJack Steiner mutex_lock(>s->ts_ctxlock); 945fe5bb6b0SJack Steiner preempt_disable(); 946364b76dfSJack Steiner 94755484c45SJack Steiner gru_check_context_placement(gts); 9489a0deeccSJack Steiner 9499a0deeccSJack Steiner if (!gts->ts_gru) { 950836ce679SJack Steiner STAT(load_user_context); 95155484c45SJack Steiner if (!gru_assign_gru_context(gts)) { 9529a0deeccSJack Steiner preempt_enable(); 953364b76dfSJack Steiner mutex_unlock(>s->ts_ctxlock); 954364b76dfSJack Steiner set_current_state(TASK_INTERRUPTIBLE); 9559a0deeccSJack Steiner schedule_timeout(GRU_ASSIGN_DELAY); /* true hack ZZZ */ 9569a0deeccSJack Steiner if (gts->ts_steal_jiffies + GRU_STEAL_DELAY < jiffies) 95755484c45SJack Steiner gru_steal_context(gts); 9589a0deeccSJack Steiner goto again; 9599a0deeccSJack Steiner } 9609a0deeccSJack Steiner gru_load_context(gts); 9619a0deeccSJack Steiner paddr = gseg_physical_address(gts->ts_gru, gts->ts_ctxnum); 9629a0deeccSJack Steiner remap_pfn_range(vma, vaddr & ~(GRU_GSEG_PAGESIZE - 1), 9639a0deeccSJack Steiner paddr >> PAGE_SHIFT, GRU_GSEG_PAGESIZE, 9649a0deeccSJack Steiner vma->vm_page_prot); 9659a0deeccSJack Steiner } 9669a0deeccSJack Steiner 9679a0deeccSJack Steiner preempt_enable(); 968364b76dfSJack Steiner mutex_unlock(>s->ts_ctxlock); 9699a0deeccSJack Steiner 9709a0deeccSJack Steiner return VM_FAULT_NOPAGE; 9719a0deeccSJack Steiner } 9729a0deeccSJack Steiner 973