113d19498SJack Steiner /* 213d19498SJack Steiner * SN Platform GRU Driver 313d19498SJack Steiner * 413d19498SJack Steiner * GRU DRIVER TABLES, MACROS, externs, etc 513d19498SJack Steiner * 613d19498SJack Steiner * Copyright (c) 2008 Silicon Graphics, Inc. All Rights Reserved. 713d19498SJack Steiner * 813d19498SJack Steiner * This program is free software; you can redistribute it and/or modify 913d19498SJack Steiner * it under the terms of the GNU General Public License as published by 1013d19498SJack Steiner * the Free Software Foundation; either version 2 of the License, or 1113d19498SJack Steiner * (at your option) any later version. 1213d19498SJack Steiner * 1313d19498SJack Steiner * This program is distributed in the hope that it will be useful, 1413d19498SJack Steiner * but WITHOUT ANY WARRANTY; without even the implied warranty of 1513d19498SJack Steiner * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 1613d19498SJack Steiner * GNU General Public License for more details. 1713d19498SJack Steiner * 1813d19498SJack Steiner * You should have received a copy of the GNU General Public License 1913d19498SJack Steiner * along with this program; if not, write to the Free Software 2013d19498SJack Steiner * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 2113d19498SJack Steiner */ 2213d19498SJack Steiner 2313d19498SJack Steiner #ifndef __GRUTABLES_H__ 2413d19498SJack Steiner #define __GRUTABLES_H__ 2513d19498SJack Steiner 2613d19498SJack Steiner /* 2713d19498SJack Steiner * Tables: 2813d19498SJack Steiner * 2913d19498SJack Steiner * VDATA-VMA Data - Holds a few parameters. Head of linked list of 3013d19498SJack Steiner * GTS tables for threads using the GSEG 3113d19498SJack Steiner * GTS - Gru Thread State - contains info for managing a GSEG context. A 3213d19498SJack Steiner * GTS is allocated for each thread accessing a 3313d19498SJack Steiner * GSEG. 3413d19498SJack Steiner * GTD - GRU Thread Data - contains shadow copy of GRU data when GSEG is 3513d19498SJack Steiner * not loaded into a GRU 3613d19498SJack Steiner * GMS - GRU Memory Struct - Used to manage TLB shootdowns. Tracks GRUs 3713d19498SJack Steiner * where a GSEG has been loaded. Similar to 3813d19498SJack Steiner * an mm_struct but for GRU. 3913d19498SJack Steiner * 4013d19498SJack Steiner * GS - GRU State - Used to manage the state of a GRU chiplet 4113d19498SJack Steiner * BS - Blade State - Used to manage state of all GRU chiplets 4213d19498SJack Steiner * on a blade 4313d19498SJack Steiner * 4413d19498SJack Steiner * 4513d19498SJack Steiner * Normal task tables for task using GRU. 4613d19498SJack Steiner * - 2 threads in process 4713d19498SJack Steiner * - 2 GSEGs open in process 4813d19498SJack Steiner * - GSEG1 is being used by both threads 4913d19498SJack Steiner * - GSEG2 is used only by thread 2 5013d19498SJack Steiner * 5113d19498SJack Steiner * task -->| 5213d19498SJack Steiner * task ---+---> mm ->------ (notifier) -------+-> gms 5313d19498SJack Steiner * | | 5413d19498SJack Steiner * |--> vma -> vdata ---> gts--->| GSEG1 (thread1) 5513d19498SJack Steiner * | | | 5613d19498SJack Steiner * | +-> gts--->| GSEG1 (thread2) 5713d19498SJack Steiner * | | 5813d19498SJack Steiner * |--> vma -> vdata ---> gts--->| GSEG2 (thread2) 5913d19498SJack Steiner * . 6013d19498SJack Steiner * . 6113d19498SJack Steiner * 6213d19498SJack Steiner * GSEGs are marked DONTCOPY on fork 6313d19498SJack Steiner * 6413d19498SJack Steiner * At open 6513d19498SJack Steiner * file.private_data -> NULL 6613d19498SJack Steiner * 6713d19498SJack Steiner * At mmap, 6813d19498SJack Steiner * vma -> vdata 6913d19498SJack Steiner * 7013d19498SJack Steiner * After gseg reference 7113d19498SJack Steiner * vma -> vdata ->gts 7213d19498SJack Steiner * 7313d19498SJack Steiner * After fork 7413d19498SJack Steiner * parent 7513d19498SJack Steiner * vma -> vdata -> gts 7613d19498SJack Steiner * child 7713d19498SJack Steiner * (vma is not copied) 7813d19498SJack Steiner * 7913d19498SJack Steiner */ 8013d19498SJack Steiner 8113d19498SJack Steiner #include <linux/rmap.h> 8213d19498SJack Steiner #include <linux/interrupt.h> 8313d19498SJack Steiner #include <linux/mutex.h> 8413d19498SJack Steiner #include <linux/wait.h> 8513d19498SJack Steiner #include <linux/mmu_notifier.h> 8613d19498SJack Steiner #include "gru.h" 8713d19498SJack Steiner #include "gruhandles.h" 8813d19498SJack Steiner 8913d19498SJack Steiner extern struct gru_stats_s gru_stats; 9013d19498SJack Steiner extern struct gru_blade_state *gru_base[]; 9113d19498SJack Steiner extern unsigned long gru_start_paddr, gru_end_paddr; 9213d19498SJack Steiner 9313d19498SJack Steiner #define GRU_MAX_BLADES MAX_NUMNODES 9413d19498SJack Steiner #define GRU_MAX_GRUS (GRU_MAX_BLADES * GRU_CHIPLETS_PER_BLADE) 9513d19498SJack Steiner 9613d19498SJack Steiner #define GRU_DRIVER_ID_STR "SGI GRU Device Driver" 9713d19498SJack Steiner #define GRU_DRIVER_VERSION_STR "0.80" 9813d19498SJack Steiner 9913d19498SJack Steiner /* 10013d19498SJack Steiner * GRU statistics. 10113d19498SJack Steiner */ 10213d19498SJack Steiner struct gru_stats_s { 10313d19498SJack Steiner atomic_long_t vdata_alloc; 10413d19498SJack Steiner atomic_long_t vdata_free; 10513d19498SJack Steiner atomic_long_t gts_alloc; 10613d19498SJack Steiner atomic_long_t gts_free; 10713d19498SJack Steiner atomic_long_t vdata_double_alloc; 10813d19498SJack Steiner atomic_long_t gts_double_allocate; 10913d19498SJack Steiner atomic_long_t assign_context; 11013d19498SJack Steiner atomic_long_t assign_context_failed; 11113d19498SJack Steiner atomic_long_t free_context; 11213d19498SJack Steiner atomic_long_t load_context; 11313d19498SJack Steiner atomic_long_t unload_context; 11413d19498SJack Steiner atomic_long_t steal_context; 11513d19498SJack Steiner atomic_long_t steal_context_failed; 11613d19498SJack Steiner atomic_long_t nopfn; 11713d19498SJack Steiner atomic_long_t break_cow; 11813d19498SJack Steiner atomic_long_t asid_new; 11913d19498SJack Steiner atomic_long_t asid_next; 12013d19498SJack Steiner atomic_long_t asid_wrap; 12113d19498SJack Steiner atomic_long_t asid_reuse; 12213d19498SJack Steiner atomic_long_t intr; 12313d19498SJack Steiner atomic_long_t call_os; 12413d19498SJack Steiner atomic_long_t call_os_check_for_bug; 12513d19498SJack Steiner atomic_long_t call_os_wait_queue; 12613d19498SJack Steiner atomic_long_t user_flush_tlb; 12713d19498SJack Steiner atomic_long_t user_unload_context; 12813d19498SJack Steiner atomic_long_t user_exception; 12913d19498SJack Steiner atomic_long_t set_task_slice; 13013d19498SJack Steiner atomic_long_t migrate_check; 13113d19498SJack Steiner atomic_long_t migrated_retarget; 13213d19498SJack Steiner atomic_long_t migrated_unload; 13313d19498SJack Steiner atomic_long_t migrated_unload_delay; 13413d19498SJack Steiner atomic_long_t migrated_nopfn_retarget; 13513d19498SJack Steiner atomic_long_t migrated_nopfn_unload; 13613d19498SJack Steiner atomic_long_t tlb_dropin; 13713d19498SJack Steiner atomic_long_t tlb_dropin_fail_no_asid; 13813d19498SJack Steiner atomic_long_t tlb_dropin_fail_upm; 13913d19498SJack Steiner atomic_long_t tlb_dropin_fail_invalid; 14013d19498SJack Steiner atomic_long_t tlb_dropin_fail_range_active; 14113d19498SJack Steiner atomic_long_t tlb_dropin_fail_idle; 14213d19498SJack Steiner atomic_long_t tlb_dropin_fail_fmm; 14313d19498SJack Steiner atomic_long_t mmu_invalidate_range; 14413d19498SJack Steiner atomic_long_t mmu_invalidate_page; 14513d19498SJack Steiner atomic_long_t mmu_clear_flush_young; 14613d19498SJack Steiner atomic_long_t flush_tlb; 14713d19498SJack Steiner atomic_long_t flush_tlb_gru; 14813d19498SJack Steiner atomic_long_t flush_tlb_gru_tgh; 14913d19498SJack Steiner atomic_long_t flush_tlb_gru_zero_asid; 15013d19498SJack Steiner 15113d19498SJack Steiner atomic_long_t copy_gpa; 15213d19498SJack Steiner 15313d19498SJack Steiner atomic_long_t mesq_receive; 15413d19498SJack Steiner atomic_long_t mesq_receive_none; 15513d19498SJack Steiner atomic_long_t mesq_send; 15613d19498SJack Steiner atomic_long_t mesq_send_failed; 15713d19498SJack Steiner atomic_long_t mesq_noop; 15813d19498SJack Steiner atomic_long_t mesq_send_unexpected_error; 15913d19498SJack Steiner atomic_long_t mesq_send_lb_overflow; 16013d19498SJack Steiner atomic_long_t mesq_send_qlimit_reached; 16113d19498SJack Steiner atomic_long_t mesq_send_amo_nacked; 16213d19498SJack Steiner atomic_long_t mesq_send_put_nacked; 16313d19498SJack Steiner atomic_long_t mesq_qf_not_full; 16413d19498SJack Steiner atomic_long_t mesq_qf_locked; 16513d19498SJack Steiner atomic_long_t mesq_qf_noop_not_full; 16613d19498SJack Steiner atomic_long_t mesq_qf_switch_head_failed; 16713d19498SJack Steiner atomic_long_t mesq_qf_unexpected_error; 16813d19498SJack Steiner atomic_long_t mesq_noop_unexpected_error; 16913d19498SJack Steiner atomic_long_t mesq_noop_lb_overflow; 17013d19498SJack Steiner atomic_long_t mesq_noop_qlimit_reached; 17113d19498SJack Steiner atomic_long_t mesq_noop_amo_nacked; 17213d19498SJack Steiner atomic_long_t mesq_noop_put_nacked; 17313d19498SJack Steiner 17413d19498SJack Steiner }; 17513d19498SJack Steiner 17613d19498SJack Steiner #define OPT_DPRINT 1 17713d19498SJack Steiner #define OPT_STATS 2 17813d19498SJack Steiner #define GRU_QUICKLOOK 4 17913d19498SJack Steiner 18013d19498SJack Steiner 18113d19498SJack Steiner #define IRQ_GRU 110 /* Starting IRQ number for interrupts */ 18213d19498SJack Steiner 18313d19498SJack Steiner /* Delay in jiffies between attempts to assign a GRU context */ 18413d19498SJack Steiner #define GRU_ASSIGN_DELAY ((HZ * 20) / 1000) 18513d19498SJack Steiner 18613d19498SJack Steiner /* 18713d19498SJack Steiner * If a process has it's context stolen, min delay in jiffies before trying to 18813d19498SJack Steiner * steal a context from another process. 18913d19498SJack Steiner */ 19013d19498SJack Steiner #define GRU_STEAL_DELAY ((HZ * 200) / 1000) 19113d19498SJack Steiner 19213d19498SJack Steiner #define STAT(id) do { \ 19313d19498SJack Steiner if (options & OPT_STATS) \ 19413d19498SJack Steiner atomic_long_inc(&gru_stats.id); \ 19513d19498SJack Steiner } while (0) 19613d19498SJack Steiner 19713d19498SJack Steiner #ifdef CONFIG_SGI_GRU_DEBUG 19813d19498SJack Steiner #define gru_dbg(dev, fmt, x...) \ 19913d19498SJack Steiner do { \ 20013d19498SJack Steiner if (options & OPT_DPRINT) \ 20113d19498SJack Steiner dev_dbg(dev, "%s: " fmt, __func__, x); \ 20213d19498SJack Steiner } while (0) 20313d19498SJack Steiner #else 20413d19498SJack Steiner #define gru_dbg(x...) 20513d19498SJack Steiner #endif 20613d19498SJack Steiner 20713d19498SJack Steiner /*----------------------------------------------------------------------------- 20813d19498SJack Steiner * ASID management 20913d19498SJack Steiner */ 21013d19498SJack Steiner #define MAX_ASID 0xfffff0 21113d19498SJack Steiner #define MIN_ASID 8 21213d19498SJack Steiner #define ASID_INC 8 /* number of regions */ 21313d19498SJack Steiner 21413d19498SJack Steiner /* Generate a GRU asid value from a GRU base asid & a virtual address. */ 21513d19498SJack Steiner #if defined CONFIG_IA64 21613d19498SJack Steiner #define VADDR_HI_BIT 64 21713d19498SJack Steiner #define GRUREGION(addr) ((addr) >> (VADDR_HI_BIT - 3) & 3) 21813d19498SJack Steiner #elif defined __x86_64 21913d19498SJack Steiner #define VADDR_HI_BIT 48 22013d19498SJack Steiner #define GRUREGION(addr) (0) /* ZZZ could do better */ 22113d19498SJack Steiner #else 22213d19498SJack Steiner #error "Unsupported architecture" 22313d19498SJack Steiner #endif 22413d19498SJack Steiner #define GRUASID(asid, addr) ((asid) + GRUREGION(addr)) 22513d19498SJack Steiner 22613d19498SJack Steiner /*------------------------------------------------------------------------------ 22713d19498SJack Steiner * File & VMS Tables 22813d19498SJack Steiner */ 22913d19498SJack Steiner 23013d19498SJack Steiner struct gru_state; 23113d19498SJack Steiner 23213d19498SJack Steiner /* 23313d19498SJack Steiner * This structure is pointed to from the mmstruct via the notifier pointer. 23413d19498SJack Steiner * There is one of these per address space. 23513d19498SJack Steiner */ 23613d19498SJack Steiner struct gru_mm_tracker { 23713d19498SJack Steiner unsigned int mt_asid_gen; /* ASID wrap count */ 23813d19498SJack Steiner int mt_asid; /* current base ASID for gru */ 23913d19498SJack Steiner unsigned short mt_ctxbitmap; /* bitmap of contexts using 24013d19498SJack Steiner asid */ 24113d19498SJack Steiner }; 24213d19498SJack Steiner 24313d19498SJack Steiner struct gru_mm_struct { 24413d19498SJack Steiner struct mmu_notifier ms_notifier; 24513d19498SJack Steiner atomic_t ms_refcnt; 24613d19498SJack Steiner spinlock_t ms_asid_lock; /* protects ASID assignment */ 24713d19498SJack Steiner atomic_t ms_range_active;/* num range_invals active */ 24813d19498SJack Steiner char ms_released; 24913d19498SJack Steiner wait_queue_head_t ms_wait_queue; 25013d19498SJack Steiner DECLARE_BITMAP(ms_asidmap, GRU_MAX_GRUS); 25113d19498SJack Steiner struct gru_mm_tracker ms_asids[GRU_MAX_GRUS]; 25213d19498SJack Steiner }; 25313d19498SJack Steiner 25413d19498SJack Steiner /* 25513d19498SJack Steiner * One of these structures is allocated when a GSEG is mmaped. The 25613d19498SJack Steiner * structure is pointed to by the vma->vm_private_data field in the vma struct. 25713d19498SJack Steiner */ 25813d19498SJack Steiner struct gru_vma_data { 25913d19498SJack Steiner spinlock_t vd_lock; /* Serialize access to vma */ 26013d19498SJack Steiner struct list_head vd_head; /* head of linked list of gts */ 26113d19498SJack Steiner long vd_user_options;/* misc user option flags */ 26213d19498SJack Steiner int vd_cbr_au_count; 26313d19498SJack Steiner int vd_dsr_au_count; 26413d19498SJack Steiner }; 26513d19498SJack Steiner 26613d19498SJack Steiner /* 26713d19498SJack Steiner * One of these is allocated for each thread accessing a mmaped GRU. A linked 26813d19498SJack Steiner * list of these structure is hung off the struct gru_vma_data in the mm_struct. 26913d19498SJack Steiner */ 27013d19498SJack Steiner struct gru_thread_state { 27113d19498SJack Steiner struct list_head ts_next; /* list - head at vma-private */ 27213d19498SJack Steiner struct mutex ts_ctxlock; /* load/unload CTX lock */ 27313d19498SJack Steiner struct mm_struct *ts_mm; /* mm currently mapped to 27413d19498SJack Steiner context */ 27513d19498SJack Steiner struct vm_area_struct *ts_vma; /* vma of GRU context */ 27613d19498SJack Steiner struct gru_state *ts_gru; /* GRU where the context is 27713d19498SJack Steiner loaded */ 27813d19498SJack Steiner struct gru_mm_struct *ts_gms; /* asid & ioproc struct */ 27913d19498SJack Steiner unsigned long ts_cbr_map; /* map of allocated CBRs */ 28013d19498SJack Steiner unsigned long ts_dsr_map; /* map of allocated DATA 28113d19498SJack Steiner resources */ 28213d19498SJack Steiner unsigned long ts_steal_jiffies;/* jiffies when context last 28313d19498SJack Steiner stolen */ 28413d19498SJack Steiner long ts_user_options;/* misc user option flags */ 28513d19498SJack Steiner pid_t ts_tgid_owner; /* task that is using the 28613d19498SJack Steiner context - for migration */ 28713d19498SJack Steiner int ts_tsid; /* thread that owns the 28813d19498SJack Steiner structure */ 28913d19498SJack Steiner int ts_tlb_int_select;/* target cpu if interrupts 29013d19498SJack Steiner enabled */ 29113d19498SJack Steiner int ts_ctxnum; /* context number where the 29213d19498SJack Steiner context is loaded */ 29313d19498SJack Steiner atomic_t ts_refcnt; /* reference count GTS */ 29413d19498SJack Steiner unsigned char ts_dsr_au_count;/* Number of DSR resources 29513d19498SJack Steiner required for contest */ 29613d19498SJack Steiner unsigned char ts_cbr_au_count;/* Number of CBR resources 29713d19498SJack Steiner required for contest */ 29813d19498SJack Steiner char ts_force_unload;/* force context to be unloaded 29913d19498SJack Steiner after migration */ 30013d19498SJack Steiner char ts_cbr_idx[GRU_CBR_AU];/* CBR numbers of each 30113d19498SJack Steiner allocated CB */ 30213d19498SJack Steiner unsigned long ts_gdata[0]; /* save area for GRU data (CB, 30313d19498SJack Steiner DS, CBE) */ 30413d19498SJack Steiner }; 30513d19498SJack Steiner 30613d19498SJack Steiner /* 30713d19498SJack Steiner * Threaded programs actually allocate an array of GSEGs when a context is 30813d19498SJack Steiner * created. Each thread uses a separate GSEG. TSID is the index into the GSEG 30913d19498SJack Steiner * array. 31013d19498SJack Steiner */ 31113d19498SJack Steiner #define TSID(a, v) (((a) - (v)->vm_start) / GRU_GSEG_PAGESIZE) 31213d19498SJack Steiner #define UGRUADDR(gts) ((gts)->ts_vma->vm_start + \ 31313d19498SJack Steiner (gts)->ts_tsid * GRU_GSEG_PAGESIZE) 31413d19498SJack Steiner 31513d19498SJack Steiner #define NULLCTX (-1) /* if context not loaded into GRU */ 31613d19498SJack Steiner 31713d19498SJack Steiner /*----------------------------------------------------------------------------- 31813d19498SJack Steiner * GRU State Tables 31913d19498SJack Steiner */ 32013d19498SJack Steiner 32113d19498SJack Steiner /* 32213d19498SJack Steiner * One of these exists for each GRU chiplet. 32313d19498SJack Steiner */ 32413d19498SJack Steiner struct gru_state { 32513d19498SJack Steiner struct gru_blade_state *gs_blade; /* GRU state for entire 32613d19498SJack Steiner blade */ 32713d19498SJack Steiner unsigned long gs_gru_base_paddr; /* Physical address of 32813d19498SJack Steiner gru segments (64) */ 32913d19498SJack Steiner void *gs_gru_base_vaddr; /* Virtual address of 33013d19498SJack Steiner gru segments (64) */ 33113d19498SJack Steiner unsigned char gs_gid; /* unique GRU number */ 33213d19498SJack Steiner unsigned char gs_tgh_local_shift; /* used to pick TGH for 33313d19498SJack Steiner local flush */ 33413d19498SJack Steiner unsigned char gs_tgh_first_remote; /* starting TGH# for 33513d19498SJack Steiner remote flush */ 33613d19498SJack Steiner unsigned short gs_blade_id; /* blade of GRU */ 33713d19498SJack Steiner spinlock_t gs_asid_lock; /* lock used for 33813d19498SJack Steiner assigning asids */ 33913d19498SJack Steiner spinlock_t gs_lock; /* lock used for 34013d19498SJack Steiner assigning contexts */ 34113d19498SJack Steiner 34213d19498SJack Steiner /* -- the following are protected by the gs_asid_lock spinlock ---- */ 34313d19498SJack Steiner unsigned int gs_asid; /* Next availe ASID */ 34413d19498SJack Steiner unsigned int gs_asid_limit; /* Limit of available 34513d19498SJack Steiner ASIDs */ 34613d19498SJack Steiner unsigned int gs_asid_gen; /* asid generation. 34713d19498SJack Steiner Inc on wrap */ 34813d19498SJack Steiner 34913d19498SJack Steiner /* --- the following fields are protected by the gs_lock spinlock --- */ 35013d19498SJack Steiner unsigned long gs_context_map; /* bitmap to manage 35113d19498SJack Steiner contexts in use */ 35213d19498SJack Steiner unsigned long gs_cbr_map; /* bitmap to manage CB 35313d19498SJack Steiner resources */ 35413d19498SJack Steiner unsigned long gs_dsr_map; /* bitmap used to manage 35513d19498SJack Steiner DATA resources */ 35613d19498SJack Steiner unsigned int gs_reserved_cbrs; /* Number of kernel- 35713d19498SJack Steiner reserved cbrs */ 35813d19498SJack Steiner unsigned int gs_reserved_dsr_bytes; /* Bytes of kernel- 35913d19498SJack Steiner reserved dsrs */ 36013d19498SJack Steiner unsigned short gs_active_contexts; /* number of contexts 36113d19498SJack Steiner in use */ 36213d19498SJack Steiner struct gru_thread_state *gs_gts[GRU_NUM_CCH]; /* GTS currently using 36313d19498SJack Steiner the context */ 36413d19498SJack Steiner }; 36513d19498SJack Steiner 36613d19498SJack Steiner /* 36713d19498SJack Steiner * This structure contains the GRU state for all the GRUs on a blade. 36813d19498SJack Steiner */ 36913d19498SJack Steiner struct gru_blade_state { 37013d19498SJack Steiner void *kernel_cb; /* First kernel 37113d19498SJack Steiner reserved cb */ 37213d19498SJack Steiner void *kernel_dsr; /* First kernel 37313d19498SJack Steiner reserved DSR */ 37413d19498SJack Steiner /* ---- the following are protected by the bs_lock spinlock ---- */ 37513d19498SJack Steiner spinlock_t bs_lock; /* lock used for 37613d19498SJack Steiner stealing contexts */ 37713d19498SJack Steiner int bs_lru_ctxnum; /* STEAL - last context 37813d19498SJack Steiner stolen */ 37913d19498SJack Steiner struct gru_state *bs_lru_gru; /* STEAL - last gru 38013d19498SJack Steiner stolen */ 38113d19498SJack Steiner 38213d19498SJack Steiner struct gru_state bs_grus[GRU_CHIPLETS_PER_BLADE]; 38313d19498SJack Steiner }; 38413d19498SJack Steiner 38513d19498SJack Steiner /*----------------------------------------------------------------------------- 38613d19498SJack Steiner * Address Primitives 38713d19498SJack Steiner */ 38813d19498SJack Steiner #define get_tfm_for_cpu(g, c) \ 38913d19498SJack Steiner ((struct gru_tlb_fault_map *)get_tfm((g)->gs_gru_base_vaddr, (c))) 39013d19498SJack Steiner #define get_tfh_by_index(g, i) \ 39113d19498SJack Steiner ((struct gru_tlb_fault_handle *)get_tfh((g)->gs_gru_base_vaddr, (i))) 39213d19498SJack Steiner #define get_tgh_by_index(g, i) \ 39313d19498SJack Steiner ((struct gru_tlb_global_handle *)get_tgh((g)->gs_gru_base_vaddr, (i))) 39413d19498SJack Steiner #define get_cbe_by_index(g, i) \ 39513d19498SJack Steiner ((struct gru_control_block_extended *)get_cbe((g)->gs_gru_base_vaddr,\ 39613d19498SJack Steiner (i))) 39713d19498SJack Steiner 39813d19498SJack Steiner /*----------------------------------------------------------------------------- 39913d19498SJack Steiner * Useful Macros 40013d19498SJack Steiner */ 40113d19498SJack Steiner 40213d19498SJack Steiner /* Given a blade# & chiplet#, get a pointer to the GRU */ 40313d19498SJack Steiner #define get_gru(b, c) (&gru_base[b]->bs_grus[c]) 40413d19498SJack Steiner 40513d19498SJack Steiner /* Number of bytes to save/restore when unloading/loading GRU contexts */ 40613d19498SJack Steiner #define DSR_BYTES(dsr) ((dsr) * GRU_DSR_AU_BYTES) 40713d19498SJack Steiner #define CBR_BYTES(cbr) ((cbr) * GRU_HANDLE_BYTES * GRU_CBR_AU_SIZE * 2) 40813d19498SJack Steiner 40913d19498SJack Steiner /* Convert a user CB number to the actual CBRNUM */ 41013d19498SJack Steiner #define thread_cbr_number(gts, n) ((gts)->ts_cbr_idx[(n) / GRU_CBR_AU_SIZE] \ 41113d19498SJack Steiner * GRU_CBR_AU_SIZE + (n) % GRU_CBR_AU_SIZE) 41213d19498SJack Steiner 41313d19498SJack Steiner /* Convert a gid to a pointer to the GRU */ 41413d19498SJack Steiner #define GID_TO_GRU(gid) \ 41513d19498SJack Steiner (gru_base[(gid) / GRU_CHIPLETS_PER_BLADE] ? \ 41613d19498SJack Steiner (&gru_base[(gid) / GRU_CHIPLETS_PER_BLADE]-> \ 41713d19498SJack Steiner bs_grus[(gid) % GRU_CHIPLETS_PER_BLADE]) : \ 41813d19498SJack Steiner NULL) 41913d19498SJack Steiner 42013d19498SJack Steiner /* Scan all active GRUs in a GRU bitmap */ 42113d19498SJack Steiner #define for_each_gru_in_bitmap(gid, map) \ 42213d19498SJack Steiner for ((gid) = find_first_bit((map), GRU_MAX_GRUS); (gid) < GRU_MAX_GRUS;\ 42313d19498SJack Steiner (gid)++, (gid) = find_next_bit((map), GRU_MAX_GRUS, (gid))) 42413d19498SJack Steiner 42513d19498SJack Steiner /* Scan all active GRUs on a specific blade */ 42613d19498SJack Steiner #define for_each_gru_on_blade(gru, nid, i) \ 42713d19498SJack Steiner for ((gru) = gru_base[nid]->bs_grus, (i) = 0; \ 42813d19498SJack Steiner (i) < GRU_CHIPLETS_PER_BLADE; \ 42913d19498SJack Steiner (i)++, (gru)++) 43013d19498SJack Steiner 43113d19498SJack Steiner /* Scan all active GTSs on a gru. Note: must hold ss_lock to use this macro. */ 43213d19498SJack Steiner #define for_each_gts_on_gru(gts, gru, ctxnum) \ 43313d19498SJack Steiner for ((ctxnum) = 0; (ctxnum) < GRU_NUM_CCH; (ctxnum)++) \ 43413d19498SJack Steiner if (((gts) = (gru)->gs_gts[ctxnum])) 43513d19498SJack Steiner 43613d19498SJack Steiner /* Scan each CBR whose bit is set in a TFM (or copy of) */ 43713d19498SJack Steiner #define for_each_cbr_in_tfm(i, map) \ 43813d19498SJack Steiner for ((i) = find_first_bit(map, GRU_NUM_CBE); \ 43913d19498SJack Steiner (i) < GRU_NUM_CBE; \ 44013d19498SJack Steiner (i)++, (i) = find_next_bit(map, GRU_NUM_CBE, i)) 44113d19498SJack Steiner 44213d19498SJack Steiner /* Scan each CBR in a CBR bitmap. Note: multiple CBRs in an allocation unit */ 44313d19498SJack Steiner #define for_each_cbr_in_allocation_map(i, map, k) \ 44413d19498SJack Steiner for ((k) = find_first_bit(map, GRU_CBR_AU); (k) < GRU_CBR_AU; \ 44513d19498SJack Steiner (k) = find_next_bit(map, GRU_CBR_AU, (k) + 1)) \ 44613d19498SJack Steiner for ((i) = (k)*GRU_CBR_AU_SIZE; \ 44713d19498SJack Steiner (i) < ((k) + 1) * GRU_CBR_AU_SIZE; (i)++) 44813d19498SJack Steiner 44913d19498SJack Steiner /* Scan each DSR in a DSR bitmap. Note: multiple DSRs in an allocation unit */ 45013d19498SJack Steiner #define for_each_dsr_in_allocation_map(i, map, k) \ 45113d19498SJack Steiner for ((k) = find_first_bit((const unsigned long *)map, GRU_DSR_AU);\ 45213d19498SJack Steiner (k) < GRU_DSR_AU; \ 45313d19498SJack Steiner (k) = find_next_bit((const unsigned long *)map, \ 45413d19498SJack Steiner GRU_DSR_AU, (k) + 1)) \ 45513d19498SJack Steiner for ((i) = (k) * GRU_DSR_AU_CL; \ 45613d19498SJack Steiner (i) < ((k) + 1) * GRU_DSR_AU_CL; (i)++) 45713d19498SJack Steiner 45813d19498SJack Steiner #define gseg_physical_address(gru, ctxnum) \ 45913d19498SJack Steiner ((gru)->gs_gru_base_paddr + ctxnum * GRU_GSEG_STRIDE) 46013d19498SJack Steiner #define gseg_virtual_address(gru, ctxnum) \ 46113d19498SJack Steiner ((gru)->gs_gru_base_vaddr + ctxnum * GRU_GSEG_STRIDE) 46213d19498SJack Steiner 46313d19498SJack Steiner /*----------------------------------------------------------------------------- 46413d19498SJack Steiner * Lock / Unlock GRU handles 46513d19498SJack Steiner * Use the "delresp" bit in the handle as a "lock" bit. 46613d19498SJack Steiner */ 46713d19498SJack Steiner 46813d19498SJack Steiner /* Lock hierarchy checking enabled only in emulator */ 46913d19498SJack Steiner 47013d19498SJack Steiner static inline void __lock_handle(void *h) 47113d19498SJack Steiner { 47213d19498SJack Steiner while (test_and_set_bit(1, h)) 47313d19498SJack Steiner cpu_relax(); 47413d19498SJack Steiner } 47513d19498SJack Steiner 47613d19498SJack Steiner static inline void __unlock_handle(void *h) 47713d19498SJack Steiner { 47813d19498SJack Steiner clear_bit(1, h); 47913d19498SJack Steiner } 48013d19498SJack Steiner 48113d19498SJack Steiner static inline void lock_cch_handle(struct gru_context_configuration_handle *cch) 48213d19498SJack Steiner { 48313d19498SJack Steiner __lock_handle(cch); 48413d19498SJack Steiner } 48513d19498SJack Steiner 48613d19498SJack Steiner static inline void unlock_cch_handle(struct gru_context_configuration_handle 48713d19498SJack Steiner *cch) 48813d19498SJack Steiner { 48913d19498SJack Steiner __unlock_handle(cch); 49013d19498SJack Steiner } 49113d19498SJack Steiner 49213d19498SJack Steiner static inline void lock_tgh_handle(struct gru_tlb_global_handle *tgh) 49313d19498SJack Steiner { 49413d19498SJack Steiner __lock_handle(tgh); 49513d19498SJack Steiner } 49613d19498SJack Steiner 49713d19498SJack Steiner static inline void unlock_tgh_handle(struct gru_tlb_global_handle *tgh) 49813d19498SJack Steiner { 49913d19498SJack Steiner __unlock_handle(tgh); 50013d19498SJack Steiner } 50113d19498SJack Steiner 50213d19498SJack Steiner /*----------------------------------------------------------------------------- 50313d19498SJack Steiner * Function prototypes & externs 50413d19498SJack Steiner */ 50513d19498SJack Steiner struct gru_unload_context_req; 50613d19498SJack Steiner 50713d19498SJack Steiner extern struct vm_operations_struct gru_vm_ops; 50813d19498SJack Steiner extern struct device *grudev; 50913d19498SJack Steiner 51013d19498SJack Steiner extern struct gru_vma_data *gru_alloc_vma_data(struct vm_area_struct *vma, 51113d19498SJack Steiner int tsid); 51213d19498SJack Steiner extern struct gru_thread_state *gru_find_thread_state(struct vm_area_struct 51313d19498SJack Steiner *vma, int tsid); 51413d19498SJack Steiner extern struct gru_thread_state *gru_alloc_thread_state(struct vm_area_struct 51513d19498SJack Steiner *vma, int tsid); 51613d19498SJack Steiner extern void gru_unload_context(struct gru_thread_state *gts, int savestate); 51713d19498SJack Steiner extern void gts_drop(struct gru_thread_state *gts); 51813d19498SJack Steiner extern void gru_tgh_flush_init(struct gru_state *gru); 51913d19498SJack Steiner extern int gru_kservices_init(struct gru_state *gru); 52013d19498SJack Steiner extern irqreturn_t gru_intr(int irq, void *dev_id); 52113d19498SJack Steiner extern int gru_handle_user_call_os(unsigned long address); 52213d19498SJack Steiner extern int gru_user_flush_tlb(unsigned long arg); 52313d19498SJack Steiner extern int gru_user_unload_context(unsigned long arg); 52413d19498SJack Steiner extern int gru_get_exception_detail(unsigned long arg); 52513d19498SJack Steiner extern int gru_set_task_slice(long address); 52613d19498SJack Steiner extern int gru_cpu_fault_map_id(void); 52713d19498SJack Steiner extern struct vm_area_struct *gru_find_vma(unsigned long vaddr); 52813d19498SJack Steiner extern void gru_flush_all_tlb(struct gru_state *gru); 52913d19498SJack Steiner extern int gru_proc_init(void); 53013d19498SJack Steiner extern void gru_proc_exit(void); 53113d19498SJack Steiner 53213d19498SJack Steiner extern unsigned long reserve_gru_cb_resources(struct gru_state *gru, 53313d19498SJack Steiner int cbr_au_count, char *cbmap); 53413d19498SJack Steiner extern unsigned long reserve_gru_ds_resources(struct gru_state *gru, 53513d19498SJack Steiner int dsr_au_count, char *dsmap); 53613d19498SJack Steiner extern int gru_fault(struct vm_area_struct *, struct vm_fault *vmf); 53713d19498SJack Steiner extern struct gru_mm_struct *gru_register_mmu_notifier(void); 53813d19498SJack Steiner extern void gru_drop_mmu_notifier(struct gru_mm_struct *gms); 53913d19498SJack Steiner 54013d19498SJack Steiner extern void gru_flush_tlb_range(struct gru_mm_struct *gms, unsigned long start, 54113d19498SJack Steiner unsigned long len); 54213d19498SJack Steiner 54313d19498SJack Steiner extern unsigned long options; 54413d19498SJack Steiner 54513d19498SJack Steiner #endif /* __GRUTABLES_H__ */ 546