1 /* 2 * SN Platform GRU Driver 3 * 4 * FILE OPERATIONS & DRIVER INITIALIZATION 5 * 6 * This file supports the user system call for file open, close, mmap, etc. 7 * This also incudes the driver initialization code. 8 * 9 * Copyright (c) 2008 Silicon Graphics, Inc. All Rights Reserved. 10 * 11 * This program is free software; you can redistribute it and/or modify 12 * it under the terms of the GNU General Public License as published by 13 * the Free Software Foundation; either version 2 of the License, or 14 * (at your option) any later version. 15 * 16 * This program is distributed in the hope that it will be useful, 17 * but WITHOUT ANY WARRANTY; without even the implied warranty of 18 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 19 * GNU General Public License for more details. 20 * 21 * You should have received a copy of the GNU General Public License 22 * along with this program; if not, write to the Free Software 23 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 24 */ 25 26 #include <linux/module.h> 27 #include <linux/kernel.h> 28 #include <linux/errno.h> 29 #include <linux/slab.h> 30 #include <linux/mm.h> 31 #include <linux/io.h> 32 #include <linux/smp_lock.h> 33 #include <linux/spinlock.h> 34 #include <linux/device.h> 35 #include <linux/miscdevice.h> 36 #include <linux/interrupt.h> 37 #include <linux/proc_fs.h> 38 #include <linux/uaccess.h> 39 #include <asm/uv/uv.h> 40 #include "gru.h" 41 #include "grulib.h" 42 #include "grutables.h" 43 44 #include <asm/uv/uv_hub.h> 45 #include <asm/uv/uv_mmrs.h> 46 47 struct gru_blade_state *gru_base[GRU_MAX_BLADES] __read_mostly; 48 unsigned long gru_start_paddr, gru_end_paddr __read_mostly; 49 struct gru_stats_s gru_stats; 50 51 /* Guaranteed user available resources on each node */ 52 static int max_user_cbrs, max_user_dsr_bytes; 53 54 static struct file_operations gru_fops; 55 static struct miscdevice gru_miscdev; 56 57 58 /* 59 * gru_vma_close 60 * 61 * Called when unmapping a device mapping. Frees all gru resources 62 * and tables belonging to the vma. 63 */ 64 static void gru_vma_close(struct vm_area_struct *vma) 65 { 66 struct gru_vma_data *vdata; 67 struct gru_thread_state *gts; 68 struct list_head *entry, *next; 69 70 if (!vma->vm_private_data) 71 return; 72 73 vdata = vma->vm_private_data; 74 vma->vm_private_data = NULL; 75 gru_dbg(grudev, "vma %p, file %p, vdata %p\n", vma, vma->vm_file, 76 vdata); 77 list_for_each_safe(entry, next, &vdata->vd_head) { 78 gts = 79 list_entry(entry, struct gru_thread_state, ts_next); 80 list_del(>s->ts_next); 81 mutex_lock(>s->ts_ctxlock); 82 if (gts->ts_gru) 83 gru_unload_context(gts, 0); 84 mutex_unlock(>s->ts_ctxlock); 85 gts_drop(gts); 86 } 87 kfree(vdata); 88 STAT(vdata_free); 89 } 90 91 /* 92 * gru_file_mmap 93 * 94 * Called when mmaping the device. Initializes the vma with a fault handler 95 * and private data structure necessary to allocate, track, and free the 96 * underlying pages. 97 */ 98 static int gru_file_mmap(struct file *file, struct vm_area_struct *vma) 99 { 100 if ((vma->vm_flags & (VM_SHARED | VM_WRITE)) != (VM_SHARED | VM_WRITE)) 101 return -EPERM; 102 103 if (vma->vm_start & (GRU_GSEG_PAGESIZE - 1) || 104 vma->vm_end & (GRU_GSEG_PAGESIZE - 1)) 105 return -EINVAL; 106 107 vma->vm_flags |= 108 (VM_IO | VM_DONTCOPY | VM_LOCKED | VM_DONTEXPAND | VM_PFNMAP | 109 VM_RESERVED); 110 vma->vm_page_prot = PAGE_SHARED; 111 vma->vm_ops = &gru_vm_ops; 112 113 vma->vm_private_data = gru_alloc_vma_data(vma, 0); 114 if (!vma->vm_private_data) 115 return -ENOMEM; 116 117 gru_dbg(grudev, "file %p, vaddr 0x%lx, vma %p, vdata %p\n", 118 file, vma->vm_start, vma, vma->vm_private_data); 119 return 0; 120 } 121 122 /* 123 * Create a new GRU context 124 */ 125 static int gru_create_new_context(unsigned long arg) 126 { 127 struct gru_create_context_req req; 128 struct vm_area_struct *vma; 129 struct gru_vma_data *vdata; 130 int ret = -EINVAL; 131 132 133 if (copy_from_user(&req, (void __user *)arg, sizeof(req))) 134 return -EFAULT; 135 136 if (req.data_segment_bytes == 0 || 137 req.data_segment_bytes > max_user_dsr_bytes) 138 return -EINVAL; 139 if (!req.control_blocks || !req.maximum_thread_count || 140 req.control_blocks > max_user_cbrs) 141 return -EINVAL; 142 143 if (!(req.options & GRU_OPT_MISS_MASK)) 144 req.options |= GRU_OPT_MISS_FMM_INTR; 145 146 down_write(¤t->mm->mmap_sem); 147 vma = gru_find_vma(req.gseg); 148 if (vma) { 149 vdata = vma->vm_private_data; 150 vdata->vd_user_options = req.options; 151 vdata->vd_dsr_au_count = 152 GRU_DS_BYTES_TO_AU(req.data_segment_bytes); 153 vdata->vd_cbr_au_count = GRU_CB_COUNT_TO_AU(req.control_blocks); 154 ret = 0; 155 } 156 up_write(¤t->mm->mmap_sem); 157 158 return ret; 159 } 160 161 /* 162 * Get GRU configuration info (temp - for emulator testing) 163 */ 164 static long gru_get_config_info(unsigned long arg) 165 { 166 struct gru_config_info info; 167 int nodesperblade; 168 169 if (num_online_nodes() > 1 && 170 (uv_node_to_blade_id(1) == uv_node_to_blade_id(0))) 171 nodesperblade = 2; 172 else 173 nodesperblade = 1; 174 info.cpus = num_online_cpus(); 175 info.nodes = num_online_nodes(); 176 info.blades = info.nodes / nodesperblade; 177 info.chiplets = GRU_CHIPLETS_PER_BLADE * info.blades; 178 179 if (copy_to_user((void __user *)arg, &info, sizeof(info))) 180 return -EFAULT; 181 return 0; 182 } 183 184 /* 185 * Get GRU chiplet status 186 */ 187 static long gru_get_chiplet_status(unsigned long arg) 188 { 189 struct gru_state *gru; 190 struct gru_chiplet_info info; 191 192 if (copy_from_user(&info, (void __user *)arg, sizeof(info))) 193 return -EFAULT; 194 195 if (info.node == -1) 196 info.node = numa_node_id(); 197 if (info.node >= num_possible_nodes() || 198 info.chiplet >= GRU_CHIPLETS_PER_HUB || 199 info.node < 0 || info.chiplet < 0) 200 return -EINVAL; 201 202 info.blade = uv_node_to_blade_id(info.node); 203 gru = get_gru(info.blade, info.chiplet); 204 205 info.total_dsr_bytes = GRU_NUM_DSR_BYTES; 206 info.total_cbr = GRU_NUM_CB; 207 info.total_user_dsr_bytes = GRU_NUM_DSR_BYTES - 208 gru->gs_reserved_dsr_bytes; 209 info.total_user_cbr = GRU_NUM_CB - gru->gs_reserved_cbrs; 210 info.free_user_dsr_bytes = hweight64(gru->gs_dsr_map) * 211 GRU_DSR_AU_BYTES; 212 info.free_user_cbr = hweight64(gru->gs_cbr_map) * GRU_CBR_AU_SIZE; 213 214 if (copy_to_user((void __user *)arg, &info, sizeof(info))) 215 return -EFAULT; 216 return 0; 217 } 218 219 /* 220 * gru_file_unlocked_ioctl 221 * 222 * Called to update file attributes via IOCTL calls. 223 */ 224 static long gru_file_unlocked_ioctl(struct file *file, unsigned int req, 225 unsigned long arg) 226 { 227 int err = -EBADRQC; 228 229 gru_dbg(grudev, "file %p\n", file); 230 231 switch (req) { 232 case GRU_CREATE_CONTEXT: 233 err = gru_create_new_context(arg); 234 break; 235 case GRU_SET_TASK_SLICE: 236 err = gru_set_task_slice(arg); 237 break; 238 case GRU_USER_GET_EXCEPTION_DETAIL: 239 err = gru_get_exception_detail(arg); 240 break; 241 case GRU_USER_UNLOAD_CONTEXT: 242 err = gru_user_unload_context(arg); 243 break; 244 case GRU_GET_CHIPLET_STATUS: 245 err = gru_get_chiplet_status(arg); 246 break; 247 case GRU_USER_FLUSH_TLB: 248 err = gru_user_flush_tlb(arg); 249 break; 250 case GRU_USER_CALL_OS: 251 err = gru_handle_user_call_os(arg); 252 break; 253 case GRU_GET_CONFIG_INFO: 254 err = gru_get_config_info(arg); 255 break; 256 } 257 return err; 258 } 259 260 /* 261 * Called at init time to build tables for all GRUs that are present in the 262 * system. 263 */ 264 static void gru_init_chiplet(struct gru_state *gru, unsigned long paddr, 265 void *vaddr, int nid, int bid, int grunum) 266 { 267 spin_lock_init(&gru->gs_lock); 268 spin_lock_init(&gru->gs_asid_lock); 269 gru->gs_gru_base_paddr = paddr; 270 gru->gs_gru_base_vaddr = vaddr; 271 gru->gs_gid = bid * GRU_CHIPLETS_PER_BLADE + grunum; 272 gru->gs_blade = gru_base[bid]; 273 gru->gs_blade_id = bid; 274 gru->gs_cbr_map = (GRU_CBR_AU == 64) ? ~0 : (1UL << GRU_CBR_AU) - 1; 275 gru->gs_dsr_map = (1UL << GRU_DSR_AU) - 1; 276 gru_tgh_flush_init(gru); 277 gru_dbg(grudev, "bid %d, nid %d, gru %x, vaddr %p (0x%lx)\n", 278 bid, nid, gru->gs_gid, gru->gs_gru_base_vaddr, 279 gru->gs_gru_base_paddr); 280 gru_kservices_init(gru); 281 } 282 283 static int gru_init_tables(unsigned long gru_base_paddr, void *gru_base_vaddr) 284 { 285 int pnode, nid, bid, chip; 286 int cbrs, dsrbytes, n; 287 int order = get_order(sizeof(struct gru_blade_state)); 288 struct page *page; 289 struct gru_state *gru; 290 unsigned long paddr; 291 void *vaddr; 292 293 max_user_cbrs = GRU_NUM_CB; 294 max_user_dsr_bytes = GRU_NUM_DSR_BYTES; 295 for_each_online_node(nid) { 296 bid = uv_node_to_blade_id(nid); 297 pnode = uv_node_to_pnode(nid); 298 if (gru_base[bid]) 299 continue; 300 page = alloc_pages_node(nid, GFP_KERNEL, order); 301 if (!page) 302 goto fail; 303 gru_base[bid] = page_address(page); 304 memset(gru_base[bid], 0, sizeof(struct gru_blade_state)); 305 gru_base[bid]->bs_lru_gru = &gru_base[bid]->bs_grus[0]; 306 spin_lock_init(&gru_base[bid]->bs_lock); 307 308 dsrbytes = 0; 309 cbrs = 0; 310 for (gru = gru_base[bid]->bs_grus, chip = 0; 311 chip < GRU_CHIPLETS_PER_BLADE; 312 chip++, gru++) { 313 paddr = gru_chiplet_paddr(gru_base_paddr, pnode, chip); 314 vaddr = gru_chiplet_vaddr(gru_base_vaddr, pnode, chip); 315 gru_init_chiplet(gru, paddr, vaddr, bid, nid, chip); 316 n = hweight64(gru->gs_cbr_map) * GRU_CBR_AU_SIZE; 317 cbrs = max(cbrs, n); 318 n = hweight64(gru->gs_dsr_map) * GRU_DSR_AU_BYTES; 319 dsrbytes = max(dsrbytes, n); 320 } 321 max_user_cbrs = min(max_user_cbrs, cbrs); 322 max_user_dsr_bytes = min(max_user_dsr_bytes, dsrbytes); 323 } 324 325 return 0; 326 327 fail: 328 for (nid--; nid >= 0; nid--) 329 free_pages((unsigned long)gru_base[nid], order); 330 return -ENOMEM; 331 } 332 333 #ifdef CONFIG_IA64 334 335 static int get_base_irq(void) 336 { 337 return IRQ_GRU; 338 } 339 340 #elif defined CONFIG_X86_64 341 342 static void noop(unsigned int irq) 343 { 344 } 345 346 static struct irq_chip gru_chip = { 347 .name = "gru", 348 .mask = noop, 349 .unmask = noop, 350 .ack = noop, 351 }; 352 353 static int get_base_irq(void) 354 { 355 set_irq_chip(IRQ_GRU, &gru_chip); 356 set_irq_chip(IRQ_GRU + 1, &gru_chip); 357 return IRQ_GRU; 358 } 359 #endif 360 361 /* 362 * gru_init 363 * 364 * Called at boot or module load time to initialize the GRUs. 365 */ 366 static int __init gru_init(void) 367 { 368 int ret, irq, chip; 369 char id[10]; 370 void *gru_start_vaddr; 371 372 if (!is_uv_system()) 373 return 0; 374 375 #if defined CONFIG_IA64 376 gru_start_paddr = 0xd000000000UL; /* ZZZZZZZZZZZZZZZZZZZ fixme */ 377 #else 378 gru_start_paddr = uv_read_local_mmr(UVH_RH_GAM_GRU_OVERLAY_CONFIG_MMR) & 379 0x7fffffffffffUL; 380 381 #endif 382 gru_start_vaddr = __va(gru_start_paddr); 383 gru_end_paddr = gru_start_paddr + MAX_NUMNODES * GRU_SIZE; 384 printk(KERN_INFO "GRU space: 0x%lx - 0x%lx\n", 385 gru_start_paddr, gru_end_paddr); 386 irq = get_base_irq(); 387 for (chip = 0; chip < GRU_CHIPLETS_PER_BLADE; chip++) { 388 ret = request_irq(irq + chip, gru_intr, 0, id, NULL); 389 /* TODO: fix irq handling on x86. For now ignore failures because 390 * interrupts are not required & not yet fully supported */ 391 if (ret) { 392 printk("!!!WARNING: GRU ignoring request failure!!!\n"); 393 ret = 0; 394 } 395 if (ret) { 396 printk(KERN_ERR "%s: request_irq failed\n", 397 GRU_DRIVER_ID_STR); 398 goto exit1; 399 } 400 } 401 402 ret = misc_register(&gru_miscdev); 403 if (ret) { 404 printk(KERN_ERR "%s: misc_register failed\n", 405 GRU_DRIVER_ID_STR); 406 goto exit1; 407 } 408 409 ret = gru_proc_init(); 410 if (ret) { 411 printk(KERN_ERR "%s: proc init failed\n", GRU_DRIVER_ID_STR); 412 goto exit2; 413 } 414 415 ret = gru_init_tables(gru_start_paddr, gru_start_vaddr); 416 if (ret) { 417 printk(KERN_ERR "%s: init tables failed\n", GRU_DRIVER_ID_STR); 418 goto exit3; 419 } 420 421 printk(KERN_INFO "%s: v%s\n", GRU_DRIVER_ID_STR, 422 GRU_DRIVER_VERSION_STR); 423 return 0; 424 425 exit3: 426 gru_proc_exit(); 427 exit2: 428 misc_deregister(&gru_miscdev); 429 exit1: 430 for (--chip; chip >= 0; chip--) 431 free_irq(irq + chip, NULL); 432 return ret; 433 434 } 435 436 static void __exit gru_exit(void) 437 { 438 int i, bid; 439 int order = get_order(sizeof(struct gru_state) * 440 GRU_CHIPLETS_PER_BLADE); 441 442 if (!is_uv_system()) 443 return; 444 445 for (i = 0; i < GRU_CHIPLETS_PER_BLADE; i++) 446 free_irq(IRQ_GRU + i, NULL); 447 448 for (bid = 0; bid < GRU_MAX_BLADES; bid++) 449 free_pages((unsigned long)gru_base[bid], order); 450 451 misc_deregister(&gru_miscdev); 452 gru_proc_exit(); 453 } 454 455 static struct file_operations gru_fops = { 456 .owner = THIS_MODULE, 457 .unlocked_ioctl = gru_file_unlocked_ioctl, 458 .mmap = gru_file_mmap, 459 }; 460 461 static struct miscdevice gru_miscdev = { 462 .minor = MISC_DYNAMIC_MINOR, 463 .name = "gru", 464 .fops = &gru_fops, 465 }; 466 467 struct vm_operations_struct gru_vm_ops = { 468 .close = gru_vma_close, 469 .fault = gru_fault, 470 }; 471 472 fs_initcall(gru_init); 473 module_exit(gru_exit); 474 475 module_param(gru_options, ulong, 0644); 476 MODULE_PARM_DESC(gru_options, "Various debug options"); 477 478 MODULE_AUTHOR("Silicon Graphics, Inc."); 479 MODULE_LICENSE("GPL"); 480 MODULE_DESCRIPTION(GRU_DRIVER_ID_STR GRU_DRIVER_VERSION_STR); 481 MODULE_VERSION(GRU_DRIVER_VERSION_STR); 482 483