1 #include <linux/debugfs.h> 2 #include <linux/mm.h> 3 #include <linux/slab.h> 4 #include <linux/uaccess.h> 5 #include <linux/bootmem.h> 6 #include <linux/stacktrace.h> 7 #include <linux/page_owner.h> 8 #include "internal.h" 9 10 static bool page_owner_disabled = true; 11 bool page_owner_inited __read_mostly; 12 13 static void init_early_allocated_pages(void); 14 15 static int early_page_owner_param(char *buf) 16 { 17 if (!buf) 18 return -EINVAL; 19 20 if (strcmp(buf, "on") == 0) 21 page_owner_disabled = false; 22 23 return 0; 24 } 25 early_param("page_owner", early_page_owner_param); 26 27 static bool need_page_owner(void) 28 { 29 if (page_owner_disabled) 30 return false; 31 32 return true; 33 } 34 35 static void init_page_owner(void) 36 { 37 if (page_owner_disabled) 38 return; 39 40 page_owner_inited = true; 41 init_early_allocated_pages(); 42 } 43 44 struct page_ext_operations page_owner_ops = { 45 .need = need_page_owner, 46 .init = init_page_owner, 47 }; 48 49 void __reset_page_owner(struct page *page, unsigned int order) 50 { 51 int i; 52 struct page_ext *page_ext; 53 54 for (i = 0; i < (1 << order); i++) { 55 page_ext = lookup_page_ext(page + i); 56 __clear_bit(PAGE_EXT_OWNER, &page_ext->flags); 57 } 58 } 59 60 void __set_page_owner(struct page *page, unsigned int order, gfp_t gfp_mask) 61 { 62 struct page_ext *page_ext = lookup_page_ext(page); 63 struct stack_trace trace = { 64 .nr_entries = 0, 65 .max_entries = ARRAY_SIZE(page_ext->trace_entries), 66 .entries = &page_ext->trace_entries[0], 67 .skip = 3, 68 }; 69 70 save_stack_trace(&trace); 71 72 page_ext->order = order; 73 page_ext->gfp_mask = gfp_mask; 74 page_ext->nr_entries = trace.nr_entries; 75 76 __set_bit(PAGE_EXT_OWNER, &page_ext->flags); 77 } 78 79 static ssize_t 80 print_page_owner(char __user *buf, size_t count, unsigned long pfn, 81 struct page *page, struct page_ext *page_ext) 82 { 83 int ret; 84 int pageblock_mt, page_mt; 85 char *kbuf; 86 struct stack_trace trace = { 87 .nr_entries = page_ext->nr_entries, 88 .entries = &page_ext->trace_entries[0], 89 }; 90 91 kbuf = kmalloc(count, GFP_KERNEL); 92 if (!kbuf) 93 return -ENOMEM; 94 95 ret = snprintf(kbuf, count, 96 "Page allocated via order %u, mask 0x%x\n", 97 page_ext->order, page_ext->gfp_mask); 98 99 if (ret >= count) 100 goto err; 101 102 /* Print information relevant to grouping pages by mobility */ 103 pageblock_mt = get_pfnblock_migratetype(page, pfn); 104 page_mt = gfpflags_to_migratetype(page_ext->gfp_mask); 105 ret += snprintf(kbuf + ret, count - ret, 106 "PFN %lu Block %lu type %d %s Flags %s%s%s%s%s%s%s%s%s%s%s%s\n", 107 pfn, 108 pfn >> pageblock_order, 109 pageblock_mt, 110 pageblock_mt != page_mt ? "Fallback" : " ", 111 PageLocked(page) ? "K" : " ", 112 PageError(page) ? "E" : " ", 113 PageReferenced(page) ? "R" : " ", 114 PageUptodate(page) ? "U" : " ", 115 PageDirty(page) ? "D" : " ", 116 PageLRU(page) ? "L" : " ", 117 PageActive(page) ? "A" : " ", 118 PageSlab(page) ? "S" : " ", 119 PageWriteback(page) ? "W" : " ", 120 PageCompound(page) ? "C" : " ", 121 PageSwapCache(page) ? "B" : " ", 122 PageMappedToDisk(page) ? "M" : " "); 123 124 if (ret >= count) 125 goto err; 126 127 ret += snprint_stack_trace(kbuf + ret, count - ret, &trace, 0); 128 if (ret >= count) 129 goto err; 130 131 ret += snprintf(kbuf + ret, count - ret, "\n"); 132 if (ret >= count) 133 goto err; 134 135 if (copy_to_user(buf, kbuf, ret)) 136 ret = -EFAULT; 137 138 kfree(kbuf); 139 return ret; 140 141 err: 142 kfree(kbuf); 143 return -ENOMEM; 144 } 145 146 static ssize_t 147 read_page_owner(struct file *file, char __user *buf, size_t count, loff_t *ppos) 148 { 149 unsigned long pfn; 150 struct page *page; 151 struct page_ext *page_ext; 152 153 if (!page_owner_inited) 154 return -EINVAL; 155 156 page = NULL; 157 pfn = min_low_pfn + *ppos; 158 159 /* Find a valid PFN or the start of a MAX_ORDER_NR_PAGES area */ 160 while (!pfn_valid(pfn) && (pfn & (MAX_ORDER_NR_PAGES - 1)) != 0) 161 pfn++; 162 163 drain_all_pages(NULL); 164 165 /* Find an allocated page */ 166 for (; pfn < max_pfn; pfn++) { 167 /* 168 * If the new page is in a new MAX_ORDER_NR_PAGES area, 169 * validate the area as existing, skip it if not 170 */ 171 if ((pfn & (MAX_ORDER_NR_PAGES - 1)) == 0 && !pfn_valid(pfn)) { 172 pfn += MAX_ORDER_NR_PAGES - 1; 173 continue; 174 } 175 176 /* Check for holes within a MAX_ORDER area */ 177 if (!pfn_valid_within(pfn)) 178 continue; 179 180 page = pfn_to_page(pfn); 181 if (PageBuddy(page)) { 182 unsigned long freepage_order = page_order_unsafe(page); 183 184 if (freepage_order < MAX_ORDER) 185 pfn += (1UL << freepage_order) - 1; 186 continue; 187 } 188 189 page_ext = lookup_page_ext(page); 190 191 /* 192 * Some pages could be missed by concurrent allocation or free, 193 * because we don't hold the zone lock. 194 */ 195 if (!test_bit(PAGE_EXT_OWNER, &page_ext->flags)) 196 continue; 197 198 /* Record the next PFN to read in the file offset */ 199 *ppos = (pfn - min_low_pfn) + 1; 200 201 return print_page_owner(buf, count, pfn, page, page_ext); 202 } 203 204 return 0; 205 } 206 207 static void init_pages_in_zone(pg_data_t *pgdat, struct zone *zone) 208 { 209 struct page *page; 210 struct page_ext *page_ext; 211 unsigned long pfn = zone->zone_start_pfn, block_end_pfn; 212 unsigned long end_pfn = pfn + zone->spanned_pages; 213 unsigned long count = 0; 214 215 /* Scan block by block. First and last block may be incomplete */ 216 pfn = zone->zone_start_pfn; 217 218 /* 219 * Walk the zone in pageblock_nr_pages steps. If a page block spans 220 * a zone boundary, it will be double counted between zones. This does 221 * not matter as the mixed block count will still be correct 222 */ 223 for (; pfn < end_pfn; ) { 224 if (!pfn_valid(pfn)) { 225 pfn = ALIGN(pfn + 1, MAX_ORDER_NR_PAGES); 226 continue; 227 } 228 229 block_end_pfn = ALIGN(pfn + 1, pageblock_nr_pages); 230 block_end_pfn = min(block_end_pfn, end_pfn); 231 232 page = pfn_to_page(pfn); 233 234 for (; pfn < block_end_pfn; pfn++) { 235 if (!pfn_valid_within(pfn)) 236 continue; 237 238 page = pfn_to_page(pfn); 239 240 /* 241 * We are safe to check buddy flag and order, because 242 * this is init stage and only single thread runs. 243 */ 244 if (PageBuddy(page)) { 245 pfn += (1UL << page_order(page)) - 1; 246 continue; 247 } 248 249 if (PageReserved(page)) 250 continue; 251 252 page_ext = lookup_page_ext(page); 253 254 /* Maybe overraping zone */ 255 if (test_bit(PAGE_EXT_OWNER, &page_ext->flags)) 256 continue; 257 258 /* Found early allocated page */ 259 set_page_owner(page, 0, 0); 260 count++; 261 } 262 } 263 264 pr_info("Node %d, zone %8s: page owner found early allocated %lu pages\n", 265 pgdat->node_id, zone->name, count); 266 } 267 268 static void init_zones_in_node(pg_data_t *pgdat) 269 { 270 struct zone *zone; 271 struct zone *node_zones = pgdat->node_zones; 272 unsigned long flags; 273 274 for (zone = node_zones; zone - node_zones < MAX_NR_ZONES; ++zone) { 275 if (!populated_zone(zone)) 276 continue; 277 278 spin_lock_irqsave(&zone->lock, flags); 279 init_pages_in_zone(pgdat, zone); 280 spin_unlock_irqrestore(&zone->lock, flags); 281 } 282 } 283 284 static void init_early_allocated_pages(void) 285 { 286 pg_data_t *pgdat; 287 288 drain_all_pages(NULL); 289 for_each_online_pgdat(pgdat) 290 init_zones_in_node(pgdat); 291 } 292 293 static const struct file_operations proc_page_owner_operations = { 294 .read = read_page_owner, 295 }; 296 297 static int __init pageowner_init(void) 298 { 299 struct dentry *dentry; 300 301 if (!page_owner_inited) { 302 pr_info("page_owner is disabled\n"); 303 return 0; 304 } 305 306 dentry = debugfs_create_file("page_owner", S_IRUSR, NULL, 307 NULL, &proc_page_owner_operations); 308 if (IS_ERR(dentry)) 309 return PTR_ERR(dentry); 310 311 return 0; 312 } 313 module_init(pageowner_init) 314