1 /* 2 * SN Platform GRU Driver 3 * 4 * FILE OPERATIONS & DRIVER INITIALIZATION 5 * 6 * This file supports the user system call for file open, close, mmap, etc. 7 * This also incudes the driver initialization code. 8 * 9 * Copyright (c) 2008 Silicon Graphics, Inc. All Rights Reserved. 10 * 11 * This program is free software; you can redistribute it and/or modify 12 * it under the terms of the GNU General Public License as published by 13 * the Free Software Foundation; either version 2 of the License, or 14 * (at your option) any later version. 15 * 16 * This program is distributed in the hope that it will be useful, 17 * but WITHOUT ANY WARRANTY; without even the implied warranty of 18 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 19 * GNU General Public License for more details. 20 * 21 * You should have received a copy of the GNU General Public License 22 * along with this program; if not, write to the Free Software 23 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 24 */ 25 26 #include <linux/module.h> 27 #include <linux/kernel.h> 28 #include <linux/errno.h> 29 #include <linux/slab.h> 30 #include <linux/mm.h> 31 #include <linux/io.h> 32 #include <linux/smp_lock.h> 33 #include <linux/spinlock.h> 34 #include <linux/device.h> 35 #include <linux/miscdevice.h> 36 #include <linux/interrupt.h> 37 #include <linux/proc_fs.h> 38 #include <linux/uaccess.h> 39 #include <asm/uv/uv.h> 40 #include "gru.h" 41 #include "grulib.h" 42 #include "grutables.h" 43 44 #include <asm/uv/uv_hub.h> 45 #include <asm/uv/uv_mmrs.h> 46 47 struct gru_blade_state *gru_base[GRU_MAX_BLADES] __read_mostly; 48 unsigned long gru_start_paddr __read_mostly; 49 unsigned long gru_end_paddr __read_mostly; 50 unsigned int gru_max_gids __read_mostly; 51 struct gru_stats_s gru_stats; 52 53 /* Guaranteed user available resources on each node */ 54 static int max_user_cbrs, max_user_dsr_bytes; 55 56 static struct file_operations gru_fops; 57 static struct miscdevice gru_miscdev; 58 59 60 /* 61 * gru_vma_close 62 * 63 * Called when unmapping a device mapping. Frees all gru resources 64 * and tables belonging to the vma. 65 */ 66 static void gru_vma_close(struct vm_area_struct *vma) 67 { 68 struct gru_vma_data *vdata; 69 struct gru_thread_state *gts; 70 struct list_head *entry, *next; 71 72 if (!vma->vm_private_data) 73 return; 74 75 vdata = vma->vm_private_data; 76 vma->vm_private_data = NULL; 77 gru_dbg(grudev, "vma %p, file %p, vdata %p\n", vma, vma->vm_file, 78 vdata); 79 list_for_each_safe(entry, next, &vdata->vd_head) { 80 gts = 81 list_entry(entry, struct gru_thread_state, ts_next); 82 list_del(>s->ts_next); 83 mutex_lock(>s->ts_ctxlock); 84 if (gts->ts_gru) 85 gru_unload_context(gts, 0); 86 mutex_unlock(>s->ts_ctxlock); 87 gts_drop(gts); 88 } 89 kfree(vdata); 90 STAT(vdata_free); 91 } 92 93 /* 94 * gru_file_mmap 95 * 96 * Called when mmaping the device. Initializes the vma with a fault handler 97 * and private data structure necessary to allocate, track, and free the 98 * underlying pages. 99 */ 100 static int gru_file_mmap(struct file *file, struct vm_area_struct *vma) 101 { 102 if ((vma->vm_flags & (VM_SHARED | VM_WRITE)) != (VM_SHARED | VM_WRITE)) 103 return -EPERM; 104 105 if (vma->vm_start & (GRU_GSEG_PAGESIZE - 1) || 106 vma->vm_end & (GRU_GSEG_PAGESIZE - 1)) 107 return -EINVAL; 108 109 vma->vm_flags |= 110 (VM_IO | VM_DONTCOPY | VM_LOCKED | VM_DONTEXPAND | VM_PFNMAP | 111 VM_RESERVED); 112 vma->vm_page_prot = PAGE_SHARED; 113 vma->vm_ops = &gru_vm_ops; 114 115 vma->vm_private_data = gru_alloc_vma_data(vma, 0); 116 if (!vma->vm_private_data) 117 return -ENOMEM; 118 119 gru_dbg(grudev, "file %p, vaddr 0x%lx, vma %p, vdata %p\n", 120 file, vma->vm_start, vma, vma->vm_private_data); 121 return 0; 122 } 123 124 /* 125 * Create a new GRU context 126 */ 127 static int gru_create_new_context(unsigned long arg) 128 { 129 struct gru_create_context_req req; 130 struct vm_area_struct *vma; 131 struct gru_vma_data *vdata; 132 int ret = -EINVAL; 133 134 135 if (copy_from_user(&req, (void __user *)arg, sizeof(req))) 136 return -EFAULT; 137 138 if (req.data_segment_bytes == 0 || 139 req.data_segment_bytes > max_user_dsr_bytes) 140 return -EINVAL; 141 if (!req.control_blocks || !req.maximum_thread_count || 142 req.control_blocks > max_user_cbrs) 143 return -EINVAL; 144 145 if (!(req.options & GRU_OPT_MISS_MASK)) 146 req.options |= GRU_OPT_MISS_FMM_INTR; 147 148 down_write(¤t->mm->mmap_sem); 149 vma = gru_find_vma(req.gseg); 150 if (vma) { 151 vdata = vma->vm_private_data; 152 vdata->vd_user_options = req.options; 153 vdata->vd_dsr_au_count = 154 GRU_DS_BYTES_TO_AU(req.data_segment_bytes); 155 vdata->vd_cbr_au_count = GRU_CB_COUNT_TO_AU(req.control_blocks); 156 ret = 0; 157 } 158 up_write(¤t->mm->mmap_sem); 159 160 return ret; 161 } 162 163 /* 164 * Get GRU configuration info (temp - for emulator testing) 165 */ 166 static long gru_get_config_info(unsigned long arg) 167 { 168 struct gru_config_info info; 169 int nodesperblade; 170 171 if (num_online_nodes() > 1 && 172 (uv_node_to_blade_id(1) == uv_node_to_blade_id(0))) 173 nodesperblade = 2; 174 else 175 nodesperblade = 1; 176 info.cpus = num_online_cpus(); 177 info.nodes = num_online_nodes(); 178 info.blades = info.nodes / nodesperblade; 179 info.chiplets = GRU_CHIPLETS_PER_BLADE * info.blades; 180 181 if (copy_to_user((void __user *)arg, &info, sizeof(info))) 182 return -EFAULT; 183 return 0; 184 } 185 186 /* 187 * Get GRU chiplet status 188 */ 189 static long gru_get_chiplet_status(unsigned long arg) 190 { 191 struct gru_state *gru; 192 struct gru_chiplet_info info; 193 194 if (copy_from_user(&info, (void __user *)arg, sizeof(info))) 195 return -EFAULT; 196 197 if (info.node == -1) 198 info.node = numa_node_id(); 199 if (info.node >= num_possible_nodes() || 200 info.chiplet >= GRU_CHIPLETS_PER_HUB || 201 info.node < 0 || info.chiplet < 0) 202 return -EINVAL; 203 204 info.blade = uv_node_to_blade_id(info.node); 205 gru = get_gru(info.blade, info.chiplet); 206 207 info.total_dsr_bytes = GRU_NUM_DSR_BYTES; 208 info.total_cbr = GRU_NUM_CB; 209 info.total_user_dsr_bytes = GRU_NUM_DSR_BYTES - 210 gru->gs_reserved_dsr_bytes; 211 info.total_user_cbr = GRU_NUM_CB - gru->gs_reserved_cbrs; 212 info.free_user_dsr_bytes = hweight64(gru->gs_dsr_map) * 213 GRU_DSR_AU_BYTES; 214 info.free_user_cbr = hweight64(gru->gs_cbr_map) * GRU_CBR_AU_SIZE; 215 216 if (copy_to_user((void __user *)arg, &info, sizeof(info))) 217 return -EFAULT; 218 return 0; 219 } 220 221 /* 222 * gru_file_unlocked_ioctl 223 * 224 * Called to update file attributes via IOCTL calls. 225 */ 226 static long gru_file_unlocked_ioctl(struct file *file, unsigned int req, 227 unsigned long arg) 228 { 229 int err = -EBADRQC; 230 231 gru_dbg(grudev, "file %p\n", file); 232 233 switch (req) { 234 case GRU_CREATE_CONTEXT: 235 err = gru_create_new_context(arg); 236 break; 237 case GRU_SET_TASK_SLICE: 238 err = gru_set_task_slice(arg); 239 break; 240 case GRU_USER_GET_EXCEPTION_DETAIL: 241 err = gru_get_exception_detail(arg); 242 break; 243 case GRU_USER_UNLOAD_CONTEXT: 244 err = gru_user_unload_context(arg); 245 break; 246 case GRU_GET_CHIPLET_STATUS: 247 err = gru_get_chiplet_status(arg); 248 break; 249 case GRU_USER_FLUSH_TLB: 250 err = gru_user_flush_tlb(arg); 251 break; 252 case GRU_USER_CALL_OS: 253 err = gru_handle_user_call_os(arg); 254 break; 255 case GRU_GET_CONFIG_INFO: 256 err = gru_get_config_info(arg); 257 break; 258 } 259 return err; 260 } 261 262 /* 263 * Called at init time to build tables for all GRUs that are present in the 264 * system. 265 */ 266 static void gru_init_chiplet(struct gru_state *gru, unsigned long paddr, 267 void *vaddr, int nid, int bid, int grunum) 268 { 269 spin_lock_init(&gru->gs_lock); 270 spin_lock_init(&gru->gs_asid_lock); 271 gru->gs_gru_base_paddr = paddr; 272 gru->gs_gru_base_vaddr = vaddr; 273 gru->gs_gid = bid * GRU_CHIPLETS_PER_BLADE + grunum; 274 gru->gs_blade = gru_base[bid]; 275 gru->gs_blade_id = bid; 276 gru->gs_cbr_map = (GRU_CBR_AU == 64) ? ~0 : (1UL << GRU_CBR_AU) - 1; 277 gru->gs_dsr_map = (1UL << GRU_DSR_AU) - 1; 278 gru->gs_asid_limit = MAX_ASID; 279 gru_tgh_flush_init(gru); 280 if (gru->gs_gid >= gru_max_gids) 281 gru_max_gids = gru->gs_gid + 1; 282 gru_dbg(grudev, "bid %d, nid %d, gid %d, vaddr %p (0x%lx)\n", 283 bid, nid, gru->gs_gid, gru->gs_gru_base_vaddr, 284 gru->gs_gru_base_paddr); 285 gru_kservices_init(gru); 286 } 287 288 static int gru_init_tables(unsigned long gru_base_paddr, void *gru_base_vaddr) 289 { 290 int pnode, nid, bid, chip; 291 int cbrs, dsrbytes, n; 292 int order = get_order(sizeof(struct gru_blade_state)); 293 struct page *page; 294 struct gru_state *gru; 295 unsigned long paddr; 296 void *vaddr; 297 298 max_user_cbrs = GRU_NUM_CB; 299 max_user_dsr_bytes = GRU_NUM_DSR_BYTES; 300 for_each_online_node(nid) { 301 bid = uv_node_to_blade_id(nid); 302 pnode = uv_node_to_pnode(nid); 303 if (bid < 0 || gru_base[bid]) 304 continue; 305 page = alloc_pages_node(nid, GFP_KERNEL, order); 306 if (!page) 307 goto fail; 308 gru_base[bid] = page_address(page); 309 memset(gru_base[bid], 0, sizeof(struct gru_blade_state)); 310 gru_base[bid]->bs_lru_gru = &gru_base[bid]->bs_grus[0]; 311 spin_lock_init(&gru_base[bid]->bs_lock); 312 313 dsrbytes = 0; 314 cbrs = 0; 315 for (gru = gru_base[bid]->bs_grus, chip = 0; 316 chip < GRU_CHIPLETS_PER_BLADE; 317 chip++, gru++) { 318 paddr = gru_chiplet_paddr(gru_base_paddr, pnode, chip); 319 vaddr = gru_chiplet_vaddr(gru_base_vaddr, pnode, chip); 320 gru_init_chiplet(gru, paddr, vaddr, nid, bid, chip); 321 n = hweight64(gru->gs_cbr_map) * GRU_CBR_AU_SIZE; 322 cbrs = max(cbrs, n); 323 n = hweight64(gru->gs_dsr_map) * GRU_DSR_AU_BYTES; 324 dsrbytes = max(dsrbytes, n); 325 } 326 max_user_cbrs = min(max_user_cbrs, cbrs); 327 max_user_dsr_bytes = min(max_user_dsr_bytes, dsrbytes); 328 } 329 330 return 0; 331 332 fail: 333 for (nid--; nid >= 0; nid--) 334 free_pages((unsigned long)gru_base[nid], order); 335 return -ENOMEM; 336 } 337 338 #ifdef CONFIG_IA64 339 340 static int get_base_irq(void) 341 { 342 return IRQ_GRU; 343 } 344 345 #elif defined CONFIG_X86_64 346 347 static void noop(unsigned int irq) 348 { 349 } 350 351 static struct irq_chip gru_chip = { 352 .name = "gru", 353 .mask = noop, 354 .unmask = noop, 355 .ack = noop, 356 }; 357 358 static int get_base_irq(void) 359 { 360 set_irq_chip(IRQ_GRU, &gru_chip); 361 set_irq_chip(IRQ_GRU + 1, &gru_chip); 362 return IRQ_GRU; 363 } 364 #endif 365 366 /* 367 * gru_init 368 * 369 * Called at boot or module load time to initialize the GRUs. 370 */ 371 static int __init gru_init(void) 372 { 373 int ret, irq, chip; 374 char id[10]; 375 void *gru_start_vaddr; 376 377 if (!is_uv_system()) 378 return 0; 379 380 #if defined CONFIG_IA64 381 gru_start_paddr = 0xd000000000UL; /* ZZZZZZZZZZZZZZZZZZZ fixme */ 382 #else 383 gru_start_paddr = uv_read_local_mmr(UVH_RH_GAM_GRU_OVERLAY_CONFIG_MMR) & 384 0x7fffffffffffUL; 385 #endif 386 gru_start_vaddr = __va(gru_start_paddr); 387 gru_end_paddr = gru_start_paddr + GRU_MAX_BLADES * GRU_SIZE; 388 printk(KERN_INFO "GRU space: 0x%lx - 0x%lx\n", 389 gru_start_paddr, gru_end_paddr); 390 irq = get_base_irq(); 391 for (chip = 0; chip < GRU_CHIPLETS_PER_BLADE; chip++) { 392 ret = request_irq(irq + chip, gru_intr, 0, id, NULL); 393 /* TODO: fix irq handling on x86. For now ignore failure because 394 * interrupts are not required & not yet fully supported */ 395 if (ret) { 396 printk(KERN_WARNING 397 "!!!WARNING: GRU ignoring request failure!!!\n"); 398 ret = 0; 399 } 400 if (ret) { 401 printk(KERN_ERR "%s: request_irq failed\n", 402 GRU_DRIVER_ID_STR); 403 goto exit1; 404 } 405 } 406 407 ret = misc_register(&gru_miscdev); 408 if (ret) { 409 printk(KERN_ERR "%s: misc_register failed\n", 410 GRU_DRIVER_ID_STR); 411 goto exit1; 412 } 413 414 ret = gru_proc_init(); 415 if (ret) { 416 printk(KERN_ERR "%s: proc init failed\n", GRU_DRIVER_ID_STR); 417 goto exit2; 418 } 419 420 ret = gru_init_tables(gru_start_paddr, gru_start_vaddr); 421 if (ret) { 422 printk(KERN_ERR "%s: init tables failed\n", GRU_DRIVER_ID_STR); 423 goto exit3; 424 } 425 426 printk(KERN_INFO "%s: v%s\n", GRU_DRIVER_ID_STR, 427 GRU_DRIVER_VERSION_STR); 428 return 0; 429 430 exit3: 431 gru_proc_exit(); 432 exit2: 433 misc_deregister(&gru_miscdev); 434 exit1: 435 for (--chip; chip >= 0; chip--) 436 free_irq(irq + chip, NULL); 437 return ret; 438 439 } 440 441 static void __exit gru_exit(void) 442 { 443 int i, bid, gid; 444 int order = get_order(sizeof(struct gru_state) * 445 GRU_CHIPLETS_PER_BLADE); 446 447 if (!is_uv_system()) 448 return; 449 450 for (i = 0; i < GRU_CHIPLETS_PER_BLADE; i++) 451 free_irq(IRQ_GRU + i, NULL); 452 453 foreach_gid(gid) 454 gru_kservices_exit(GID_TO_GRU(gid)); 455 456 for (bid = 0; bid < GRU_MAX_BLADES; bid++) 457 free_pages((unsigned long)gru_base[bid], order); 458 459 misc_deregister(&gru_miscdev); 460 gru_proc_exit(); 461 } 462 463 static struct file_operations gru_fops = { 464 .owner = THIS_MODULE, 465 .unlocked_ioctl = gru_file_unlocked_ioctl, 466 .mmap = gru_file_mmap, 467 }; 468 469 static struct miscdevice gru_miscdev = { 470 .minor = MISC_DYNAMIC_MINOR, 471 .name = "gru", 472 .fops = &gru_fops, 473 }; 474 475 struct vm_operations_struct gru_vm_ops = { 476 .close = gru_vma_close, 477 .fault = gru_fault, 478 }; 479 480 #ifndef MODULE 481 fs_initcall(gru_init); 482 #else 483 module_init(gru_init); 484 #endif 485 module_exit(gru_exit); 486 487 module_param(gru_options, ulong, 0644); 488 MODULE_PARM_DESC(gru_options, "Various debug options"); 489 490 MODULE_AUTHOR("Silicon Graphics, Inc."); 491 MODULE_LICENSE("GPL"); 492 MODULE_DESCRIPTION(GRU_DRIVER_ID_STR GRU_DRIVER_VERSION_STR); 493 MODULE_VERSION(GRU_DRIVER_VERSION_STR); 494 495