memcontrol.c (320424c7d44f54c18df9812fd7c45f6963524002) memcontrol.c (6126891c6d4f6f4ef50323d2020635ee255a796e)
1// SPDX-License-Identifier: GPL-2.0-or-later
2/* memcontrol.c - Memory Controller
3 *
4 * Copyright IBM Corporation, 2007
5 * Author Balbir Singh <balbir@linux.vnet.ibm.com>
6 *
7 * Copyright 2007 OpenVZ SWsoft Inc
8 * Author: Pavel Emelianov <xemul@openvz.org>

--- 64 unchanged lines hidden (view full) ---

73
74struct cgroup_subsys memory_cgrp_subsys __read_mostly;
75EXPORT_SYMBOL(memory_cgrp_subsys);
76
77struct mem_cgroup *root_mem_cgroup __read_mostly;
78
79/* Active memory cgroup to use from an interrupt context */
80DEFINE_PER_CPU(struct mem_cgroup *, int_active_memcg);
1// SPDX-License-Identifier: GPL-2.0-or-later
2/* memcontrol.c - Memory Controller
3 *
4 * Copyright IBM Corporation, 2007
5 * Author Balbir Singh <balbir@linux.vnet.ibm.com>
6 *
7 * Copyright 2007 OpenVZ SWsoft Inc
8 * Author: Pavel Emelianov <xemul@openvz.org>

--- 64 unchanged lines hidden (view full) ---

73
74struct cgroup_subsys memory_cgrp_subsys __read_mostly;
75EXPORT_SYMBOL(memory_cgrp_subsys);
76
77struct mem_cgroup *root_mem_cgroup __read_mostly;
78
79/* Active memory cgroup to use from an interrupt context */
80DEFINE_PER_CPU(struct mem_cgroup *, int_active_memcg);
81EXPORT_PER_CPU_SYMBOL_GPL(int_active_memcg);
81
82/* Socket memory accounting disabled? */
82
83/* Socket memory accounting disabled? */
83static bool cgroup_memory_nosocket;
84static bool cgroup_memory_nosocket __ro_after_init;
84
85/* Kernel memory accounting disabled? */
85
86/* Kernel memory accounting disabled? */
86static bool cgroup_memory_nokmem;
87bool cgroup_memory_nokmem __ro_after_init;
87
88/* Whether the swap controller is active */
89#ifdef CONFIG_MEMCG_SWAP
88
89/* Whether the swap controller is active */
90#ifdef CONFIG_MEMCG_SWAP
90bool cgroup_memory_noswap __read_mostly;
91bool cgroup_memory_noswap __ro_after_init;
91#else
92#define cgroup_memory_noswap 1
93#endif
94
95#ifdef CONFIG_CGROUP_WRITEBACK
96static DECLARE_WAIT_QUEUE_HEAD(memcg_cgwb_frn_waitq);
97#endif
98

--- 151 unchanged lines hidden (view full) ---

250struct cgroup_subsys_state *vmpressure_to_css(struct vmpressure *vmpr)
251{
252 return &container_of(vmpr, struct mem_cgroup, vmpressure)->css;
253}
254
255#ifdef CONFIG_MEMCG_KMEM
256extern spinlock_t css_set_lock;
257
92#else
93#define cgroup_memory_noswap 1
94#endif
95
96#ifdef CONFIG_CGROUP_WRITEBACK
97static DECLARE_WAIT_QUEUE_HEAD(memcg_cgwb_frn_waitq);
98#endif
99

--- 151 unchanged lines hidden (view full) ---

251struct cgroup_subsys_state *vmpressure_to_css(struct vmpressure *vmpr)
252{
253 return &container_of(vmpr, struct mem_cgroup, vmpressure)->css;
254}
255
256#ifdef CONFIG_MEMCG_KMEM
257extern spinlock_t css_set_lock;
258
259bool mem_cgroup_kmem_disabled(void)
260{
261 return cgroup_memory_nokmem;
262}
263
258static void obj_cgroup_uncharge_pages(struct obj_cgroup *objcg,
259 unsigned int nr_pages);
260
261static void obj_cgroup_release(struct percpu_ref *ref)
262{
263 struct obj_cgroup *objcg = container_of(ref, struct obj_cgroup, refcnt);
264static void obj_cgroup_uncharge_pages(struct obj_cgroup *objcg,
265 unsigned int nr_pages);
266
267static void obj_cgroup_release(struct percpu_ref *ref)
268{
269 struct obj_cgroup *objcg = container_of(ref, struct obj_cgroup, refcnt);
264 struct mem_cgroup *memcg;
265 unsigned int nr_bytes;
266 unsigned int nr_pages;
267 unsigned long flags;
268
269 /*
270 * At this point all allocated objects are freed, and
271 * objcg->nr_charged_bytes can't have an arbitrary byte value.
272 * However, it can be PAGE_SIZE or (x * PAGE_SIZE).

--- 12 unchanged lines hidden (view full) ---

285 *
286 * In the result, nr_charged_bytes == PAGE_SIZE.
287 * This page will be uncharged in obj_cgroup_release().
288 */
289 nr_bytes = atomic_read(&objcg->nr_charged_bytes);
290 WARN_ON_ONCE(nr_bytes & (PAGE_SIZE - 1));
291 nr_pages = nr_bytes >> PAGE_SHIFT;
292
270 unsigned int nr_bytes;
271 unsigned int nr_pages;
272 unsigned long flags;
273
274 /*
275 * At this point all allocated objects are freed, and
276 * objcg->nr_charged_bytes can't have an arbitrary byte value.
277 * However, it can be PAGE_SIZE or (x * PAGE_SIZE).

--- 12 unchanged lines hidden (view full) ---

290 *
291 * In the result, nr_charged_bytes == PAGE_SIZE.
292 * This page will be uncharged in obj_cgroup_release().
293 */
294 nr_bytes = atomic_read(&objcg->nr_charged_bytes);
295 WARN_ON_ONCE(nr_bytes & (PAGE_SIZE - 1));
296 nr_pages = nr_bytes >> PAGE_SHIFT;
297
293 spin_lock_irqsave(&css_set_lock, flags);
294 memcg = obj_cgroup_memcg(objcg);
295 if (nr_pages)
296 obj_cgroup_uncharge_pages(objcg, nr_pages);
298 if (nr_pages)
299 obj_cgroup_uncharge_pages(objcg, nr_pages);
300
301 spin_lock_irqsave(&css_set_lock, flags);
297 list_del(&objcg->list);
302 list_del(&objcg->list);
298 mem_cgroup_put(memcg);
299 spin_unlock_irqrestore(&css_set_lock, flags);
300
301 percpu_ref_exit(ref);
302 kfree_rcu(objcg, rcu);
303}
304
305static struct obj_cgroup *obj_cgroup_alloc(void)
306{

--- 18 unchanged lines hidden (view full) ---

325 struct mem_cgroup *parent)
326{
327 struct obj_cgroup *objcg, *iter;
328
329 objcg = rcu_replace_pointer(memcg->objcg, NULL, true);
330
331 spin_lock_irq(&css_set_lock);
332
303 spin_unlock_irqrestore(&css_set_lock, flags);
304
305 percpu_ref_exit(ref);
306 kfree_rcu(objcg, rcu);
307}
308
309static struct obj_cgroup *obj_cgroup_alloc(void)
310{

--- 18 unchanged lines hidden (view full) ---

329 struct mem_cgroup *parent)
330{
331 struct obj_cgroup *objcg, *iter;
332
333 objcg = rcu_replace_pointer(memcg->objcg, NULL, true);
334
335 spin_lock_irq(&css_set_lock);
336
333 /* Move active objcg to the parent's list */
334 xchg(&objcg->memcg, parent);
335 css_get(&parent->css);
336 list_add(&objcg->list, &parent->objcg_list);
337
338 /* Move already reparented objcgs to the parent's list */
339 list_for_each_entry(iter, &memcg->objcg_list, list) {
340 css_get(&parent->css);
341 xchg(&iter->memcg, parent);
342 css_put(&memcg->css);
343 }
337 /* 1) Ready to reparent active objcg. */
338 list_add(&objcg->list, &memcg->objcg_list);
339 /* 2) Reparent active objcg and already reparented objcgs to parent. */
340 list_for_each_entry(iter, &memcg->objcg_list, list)
341 WRITE_ONCE(iter->memcg, parent);
342 /* 3) Move already reparented objcgs to the parent's list */
344 list_splice(&memcg->objcg_list, &parent->objcg_list);
345
346 spin_unlock_irq(&css_set_lock);
347
348 percpu_ref_kill(&objcg->refcnt);
349}
350
351/*

--- 425 unchanged lines hidden (view full) ---

777 __mod_node_page_state(pgdat, idx, val);
778 } else {
779 lruvec = mem_cgroup_lruvec(memcg, pgdat);
780 __mod_lruvec_state(lruvec, idx, val);
781 }
782 rcu_read_unlock();
783}
784
343 list_splice(&memcg->objcg_list, &parent->objcg_list);
344
345 spin_unlock_irq(&css_set_lock);
346
347 percpu_ref_kill(&objcg->refcnt);
348}
349
350/*

--- 425 unchanged lines hidden (view full) ---

776 __mod_node_page_state(pgdat, idx, val);
777 } else {
778 lruvec = mem_cgroup_lruvec(memcg, pgdat);
779 __mod_lruvec_state(lruvec, idx, val);
780 }
781 rcu_read_unlock();
782}
783
784/*
785 * mod_objcg_mlstate() may be called with irq enabled, so
786 * mod_memcg_lruvec_state() should be used.
787 */
788static inline void mod_objcg_mlstate(struct obj_cgroup *objcg,
789 struct pglist_data *pgdat,
790 enum node_stat_item idx, int nr)
791{
792 struct mem_cgroup *memcg;
793 struct lruvec *lruvec;
794
795 rcu_read_lock();
796 memcg = obj_cgroup_memcg(objcg);
797 lruvec = mem_cgroup_lruvec(memcg, pgdat);
798 mod_memcg_lruvec_state(lruvec, idx, nr);
799 rcu_read_unlock();
800}
801
785/**
786 * __count_memcg_events - account VM events in a cgroup
787 * @memcg: the memory cgroup
788 * @idx: the event item
789 * @count: the number of events that occurred
790 */
791void __count_memcg_events(struct mem_cgroup *memcg, enum vm_event_item idx,
792 unsigned long count)

--- 88 unchanged lines hidden (view full) ---

881 */
882 if (unlikely(!p))
883 return NULL;
884
885 return mem_cgroup_from_css(task_css(p, memory_cgrp_id));
886}
887EXPORT_SYMBOL(mem_cgroup_from_task);
888
802/**
803 * __count_memcg_events - account VM events in a cgroup
804 * @memcg: the memory cgroup
805 * @idx: the event item
806 * @count: the number of events that occurred
807 */
808void __count_memcg_events(struct mem_cgroup *memcg, enum vm_event_item idx,
809 unsigned long count)

--- 88 unchanged lines hidden (view full) ---

898 */
899 if (unlikely(!p))
900 return NULL;
901
902 return mem_cgroup_from_css(task_css(p, memory_cgrp_id));
903}
904EXPORT_SYMBOL(mem_cgroup_from_task);
905
906static __always_inline struct mem_cgroup *active_memcg(void)
907{
908 if (in_interrupt())
909 return this_cpu_read(int_active_memcg);
910 else
911 return current->active_memcg;
912}
913
889/**
890 * get_mem_cgroup_from_mm: Obtain a reference on given mm_struct's memcg.
891 * @mm: mm from which memcg should be extracted. It can be NULL.
892 *
914/**
915 * get_mem_cgroup_from_mm: Obtain a reference on given mm_struct's memcg.
916 * @mm: mm from which memcg should be extracted. It can be NULL.
917 *
893 * Obtain a reference on mm->memcg and returns it if successful. Otherwise
894 * root_mem_cgroup is returned. However if mem_cgroup is disabled, NULL is
895 * returned.
918 * Obtain a reference on mm->memcg and returns it if successful. If mm
919 * is NULL, then the memcg is chosen as follows:
920 * 1) The active memcg, if set.
921 * 2) current->mm->memcg, if available
922 * 3) root memcg
923 * If mem_cgroup is disabled, NULL is returned.
896 */
897struct mem_cgroup *get_mem_cgroup_from_mm(struct mm_struct *mm)
898{
899 struct mem_cgroup *memcg;
900
901 if (mem_cgroup_disabled())
902 return NULL;
903
924 */
925struct mem_cgroup *get_mem_cgroup_from_mm(struct mm_struct *mm)
926{
927 struct mem_cgroup *memcg;
928
929 if (mem_cgroup_disabled())
930 return NULL;
931
932 /*
933 * Page cache insertions can happen without an
934 * actual mm context, e.g. during disk probing
935 * on boot, loopback IO, acct() writes etc.
936 *
937 * No need to css_get on root memcg as the reference
938 * counting is disabled on the root level in the
939 * cgroup core. See CSS_NO_REF.
940 */
941 if (unlikely(!mm)) {
942 memcg = active_memcg();
943 if (unlikely(memcg)) {
944 /* remote memcg must hold a ref */
945 css_get(&memcg->css);
946 return memcg;
947 }
948 mm = current->mm;
949 if (unlikely(!mm))
950 return root_mem_cgroup;
951 }
952
904 rcu_read_lock();
905 do {
953 rcu_read_lock();
954 do {
906 /*
907 * Page cache insertions can happen without an
908 * actual mm context, e.g. during disk probing
909 * on boot, loopback IO, acct() writes etc.
910 */
911 if (unlikely(!mm))
955 memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
956 if (unlikely(!memcg))
912 memcg = root_mem_cgroup;
957 memcg = root_mem_cgroup;
913 else {
914 memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
915 if (unlikely(!memcg))
916 memcg = root_mem_cgroup;
917 }
918 } while (!css_tryget(&memcg->css));
919 rcu_read_unlock();
920 return memcg;
921}
922EXPORT_SYMBOL(get_mem_cgroup_from_mm);
923
958 } while (!css_tryget(&memcg->css));
959 rcu_read_unlock();
960 return memcg;
961}
962EXPORT_SYMBOL(get_mem_cgroup_from_mm);
963
924static __always_inline struct mem_cgroup *active_memcg(void)
925{
926 if (in_interrupt())
927 return this_cpu_read(int_active_memcg);
928 else
929 return current->active_memcg;
930}
931
932static __always_inline bool memcg_kmem_bypass(void)
933{
934 /* Allow remote memcg charging from any context. */
935 if (unlikely(active_memcg()))
936 return false;
937
938 /* Memcg to charge can't be determined. */
964static __always_inline bool memcg_kmem_bypass(void)
965{
966 /* Allow remote memcg charging from any context. */
967 if (unlikely(active_memcg()))
968 return false;
969
970 /* Memcg to charge can't be determined. */
939 if (in_interrupt() || !current->mm || (current->flags & PF_KTHREAD))
971 if (!in_task() || !current->mm || (current->flags & PF_KTHREAD))
940 return true;
941
942 return false;
943}
944
945/**
946 * mem_cgroup_iter - iterate over memory cgroup hierarchy
947 * @root: hierarchy root

--- 225 unchanged lines hidden (view full) ---

1173 * - page locked
1174 * - PageLRU cleared
1175 * - lock_page_memcg()
1176 * - page->_refcount is zero
1177 */
1178struct lruvec *lock_page_lruvec(struct page *page)
1179{
1180 struct lruvec *lruvec;
972 return true;
973
974 return false;
975}
976
977/**
978 * mem_cgroup_iter - iterate over memory cgroup hierarchy
979 * @root: hierarchy root

--- 225 unchanged lines hidden (view full) ---

1205 * - page locked
1206 * - PageLRU cleared
1207 * - lock_page_memcg()
1208 * - page->_refcount is zero
1209 */
1210struct lruvec *lock_page_lruvec(struct page *page)
1211{
1212 struct lruvec *lruvec;
1181 struct pglist_data *pgdat = page_pgdat(page);
1182
1213
1183 lruvec = mem_cgroup_page_lruvec(page, pgdat);
1214 lruvec = mem_cgroup_page_lruvec(page);
1184 spin_lock(&lruvec->lru_lock);
1185
1186 lruvec_memcg_debug(lruvec, page);
1187
1188 return lruvec;
1189}
1190
1191struct lruvec *lock_page_lruvec_irq(struct page *page)
1192{
1193 struct lruvec *lruvec;
1215 spin_lock(&lruvec->lru_lock);
1216
1217 lruvec_memcg_debug(lruvec, page);
1218
1219 return lruvec;
1220}
1221
1222struct lruvec *lock_page_lruvec_irq(struct page *page)
1223{
1224 struct lruvec *lruvec;
1194 struct pglist_data *pgdat = page_pgdat(page);
1195
1225
1196 lruvec = mem_cgroup_page_lruvec(page, pgdat);
1226 lruvec = mem_cgroup_page_lruvec(page);
1197 spin_lock_irq(&lruvec->lru_lock);
1198
1199 lruvec_memcg_debug(lruvec, page);
1200
1201 return lruvec;
1202}
1203
1204struct lruvec *lock_page_lruvec_irqsave(struct page *page, unsigned long *flags)
1205{
1206 struct lruvec *lruvec;
1227 spin_lock_irq(&lruvec->lru_lock);
1228
1229 lruvec_memcg_debug(lruvec, page);
1230
1231 return lruvec;
1232}
1233
1234struct lruvec *lock_page_lruvec_irqsave(struct page *page, unsigned long *flags)
1235{
1236 struct lruvec *lruvec;
1207 struct pglist_data *pgdat = page_pgdat(page);
1208
1237
1209 lruvec = mem_cgroup_page_lruvec(page, pgdat);
1238 lruvec = mem_cgroup_page_lruvec(page);
1210 spin_lock_irqsave(&lruvec->lru_lock, *flags);
1211
1212 lruvec_memcg_debug(lruvec, page);
1213
1214 return lruvec;
1215}
1216
1217/**

--- 817 unchanged lines hidden (view full) ---

2035void unlock_page_memcg(struct page *page)
2036{
2037 struct page *head = compound_head(page);
2038
2039 __unlock_page_memcg(page_memcg(head));
2040}
2041EXPORT_SYMBOL(unlock_page_memcg);
2042
1239 spin_lock_irqsave(&lruvec->lru_lock, *flags);
1240
1241 lruvec_memcg_debug(lruvec, page);
1242
1243 return lruvec;
1244}
1245
1246/**

--- 817 unchanged lines hidden (view full) ---

2064void unlock_page_memcg(struct page *page)
2065{
2066 struct page *head = compound_head(page);
2067
2068 __unlock_page_memcg(page_memcg(head));
2069}
2070EXPORT_SYMBOL(unlock_page_memcg);
2071
2043struct memcg_stock_pcp {
2044 struct mem_cgroup *cached; /* this never be root cgroup */
2045 unsigned int nr_pages;
2046
2072struct obj_stock {
2047#ifdef CONFIG_MEMCG_KMEM
2048 struct obj_cgroup *cached_objcg;
2073#ifdef CONFIG_MEMCG_KMEM
2074 struct obj_cgroup *cached_objcg;
2075 struct pglist_data *cached_pgdat;
2049 unsigned int nr_bytes;
2076 unsigned int nr_bytes;
2077 int nr_slab_reclaimable_b;
2078 int nr_slab_unreclaimable_b;
2079#else
2080 int dummy[0];
2050#endif
2081#endif
2082};
2051
2083
2084struct memcg_stock_pcp {
2085 struct mem_cgroup *cached; /* this never be root cgroup */
2086 unsigned int nr_pages;
2087 struct obj_stock task_obj;
2088 struct obj_stock irq_obj;
2089
2052 struct work_struct work;
2053 unsigned long flags;
2054#define FLUSHING_CACHED_CHARGE 0
2055};
2056static DEFINE_PER_CPU(struct memcg_stock_pcp, memcg_stock);
2057static DEFINE_MUTEX(percpu_charge_mutex);
2058
2059#ifdef CONFIG_MEMCG_KMEM
2090 struct work_struct work;
2091 unsigned long flags;
2092#define FLUSHING_CACHED_CHARGE 0
2093};
2094static DEFINE_PER_CPU(struct memcg_stock_pcp, memcg_stock);
2095static DEFINE_MUTEX(percpu_charge_mutex);
2096
2097#ifdef CONFIG_MEMCG_KMEM
2060static void drain_obj_stock(struct memcg_stock_pcp *stock);
2098static void drain_obj_stock(struct obj_stock *stock);
2061static bool obj_stock_flush_required(struct memcg_stock_pcp *stock,
2062 struct mem_cgroup *root_memcg);
2063
2064#else
2099static bool obj_stock_flush_required(struct memcg_stock_pcp *stock,
2100 struct mem_cgroup *root_memcg);
2101
2102#else
2065static inline void drain_obj_stock(struct memcg_stock_pcp *stock)
2103static inline void drain_obj_stock(struct obj_stock *stock)
2066{
2067}
2068static bool obj_stock_flush_required(struct memcg_stock_pcp *stock,
2069 struct mem_cgroup *root_memcg)
2070{
2071 return false;
2072}
2073#endif
2074
2104{
2105}
2106static bool obj_stock_flush_required(struct memcg_stock_pcp *stock,
2107 struct mem_cgroup *root_memcg)
2108{
2109 return false;
2110}
2111#endif
2112
2113/*
2114 * Most kmem_cache_alloc() calls are from user context. The irq disable/enable
2115 * sequence used in this case to access content from object stock is slow.
2116 * To optimize for user context access, there are now two object stocks for
2117 * task context and interrupt context access respectively.
2118 *
2119 * The task context object stock can be accessed by disabling preemption only
2120 * which is cheap in non-preempt kernel. The interrupt context object stock
2121 * can only be accessed after disabling interrupt. User context code can
2122 * access interrupt object stock, but not vice versa.
2123 */
2124static inline struct obj_stock *get_obj_stock(unsigned long *pflags)
2125{
2126 struct memcg_stock_pcp *stock;
2127
2128 if (likely(in_task())) {
2129 *pflags = 0UL;
2130 preempt_disable();
2131 stock = this_cpu_ptr(&memcg_stock);
2132 return &stock->task_obj;
2133 }
2134
2135 local_irq_save(*pflags);
2136 stock = this_cpu_ptr(&memcg_stock);
2137 return &stock->irq_obj;
2138}
2139
2140static inline void put_obj_stock(unsigned long flags)
2141{
2142 if (likely(in_task()))
2143 preempt_enable();
2144 else
2145 local_irq_restore(flags);
2146}
2147
2075/**
2076 * consume_stock: Try to consume stocked charge on this cpu.
2077 * @memcg: memcg to consume from.
2078 * @nr_pages: how many pages to charge.
2079 *
2080 * The charges will only happen if @memcg matches the current cpu's memcg
2081 * stock, and at least @nr_pages are available in that stock. Failure to
2082 * service an allocation will refill the stock.

--- 50 unchanged lines hidden (view full) ---

2133
2134 /*
2135 * The only protection from memory hotplug vs. drain_stock races is
2136 * that we always operate on local CPU stock here with IRQ disabled
2137 */
2138 local_irq_save(flags);
2139
2140 stock = this_cpu_ptr(&memcg_stock);
2148/**
2149 * consume_stock: Try to consume stocked charge on this cpu.
2150 * @memcg: memcg to consume from.
2151 * @nr_pages: how many pages to charge.
2152 *
2153 * The charges will only happen if @memcg matches the current cpu's memcg
2154 * stock, and at least @nr_pages are available in that stock. Failure to
2155 * service an allocation will refill the stock.

--- 50 unchanged lines hidden (view full) ---

2206
2207 /*
2208 * The only protection from memory hotplug vs. drain_stock races is
2209 * that we always operate on local CPU stock here with IRQ disabled
2210 */
2211 local_irq_save(flags);
2212
2213 stock = this_cpu_ptr(&memcg_stock);
2141 drain_obj_stock(stock);
2214 drain_obj_stock(&stock->irq_obj);
2215 if (in_task())
2216 drain_obj_stock(&stock->task_obj);
2142 drain_stock(stock);
2143 clear_bit(FLUSHING_CACHED_CHARGE, &stock->flags);
2144
2145 local_irq_restore(flags);
2146}
2147
2148/*
2149 * Cache charges(val) to local per_cpu area.

--- 349 unchanged lines hidden (view full) ---

2499 psi_memstall_enter(&pflags);
2500 schedule_timeout_killable(penalty_jiffies);
2501 psi_memstall_leave(&pflags);
2502
2503out:
2504 css_put(&memcg->css);
2505}
2506
2217 drain_stock(stock);
2218 clear_bit(FLUSHING_CACHED_CHARGE, &stock->flags);
2219
2220 local_irq_restore(flags);
2221}
2222
2223/*
2224 * Cache charges(val) to local per_cpu area.

--- 349 unchanged lines hidden (view full) ---

2574 psi_memstall_enter(&pflags);
2575 schedule_timeout_killable(penalty_jiffies);
2576 psi_memstall_leave(&pflags);
2577
2578out:
2579 css_put(&memcg->css);
2580}
2581
2507static int try_charge(struct mem_cgroup *memcg, gfp_t gfp_mask,
2508 unsigned int nr_pages)
2582static int try_charge_memcg(struct mem_cgroup *memcg, gfp_t gfp_mask,
2583 unsigned int nr_pages)
2509{
2510 unsigned int batch = max(MEMCG_CHARGE_BATCH, nr_pages);
2511 int nr_retries = MAX_RECLAIM_RETRIES;
2512 struct mem_cgroup *mem_over_limit;
2513 struct page_counter *counter;
2514 enum oom_status oom_status;
2515 unsigned long nr_reclaimed;
2516 bool may_swap = true;
2517 bool drained = false;
2518 unsigned long pflags;
2519
2584{
2585 unsigned int batch = max(MEMCG_CHARGE_BATCH, nr_pages);
2586 int nr_retries = MAX_RECLAIM_RETRIES;
2587 struct mem_cgroup *mem_over_limit;
2588 struct page_counter *counter;
2589 enum oom_status oom_status;
2590 unsigned long nr_reclaimed;
2591 bool may_swap = true;
2592 bool drained = false;
2593 unsigned long pflags;
2594
2520 if (mem_cgroup_is_root(memcg))
2521 return 0;
2522retry:
2523 if (consume_stock(memcg, nr_pages))
2524 return 0;
2525
2526 if (!do_memsw_account() ||
2527 page_counter_try_charge(&memcg->memsw, batch, &counter)) {
2528 if (page_counter_try_charge(&memcg->memory, batch, &counter))
2529 goto done_restock;

--- 163 unchanged lines hidden (view full) ---

2693 set_notify_resume(current);
2694 break;
2695 }
2696 } while ((memcg = parent_mem_cgroup(memcg)));
2697
2698 return 0;
2699}
2700
2595retry:
2596 if (consume_stock(memcg, nr_pages))
2597 return 0;
2598
2599 if (!do_memsw_account() ||
2600 page_counter_try_charge(&memcg->memsw, batch, &counter)) {
2601 if (page_counter_try_charge(&memcg->memory, batch, &counter))
2602 goto done_restock;

--- 163 unchanged lines hidden (view full) ---

2766 set_notify_resume(current);
2767 break;
2768 }
2769 } while ((memcg = parent_mem_cgroup(memcg)));
2770
2771 return 0;
2772}
2773
2774static inline int try_charge(struct mem_cgroup *memcg, gfp_t gfp_mask,
2775 unsigned int nr_pages)
2776{
2777 if (mem_cgroup_is_root(memcg))
2778 return 0;
2779
2780 return try_charge_memcg(memcg, gfp_mask, nr_pages);
2781}
2782
2701#if defined(CONFIG_MEMCG_KMEM) || defined(CONFIG_MMU)
2702static void cancel_charge(struct mem_cgroup *memcg, unsigned int nr_pages)
2703{
2704 if (mem_cgroup_is_root(memcg))
2705 return;
2706
2707 page_counter_uncharge(&memcg->memory, nr_pages);
2708 if (do_memsw_account())

--- 25 unchanged lines hidden (view full) ---

2734 if (unlikely(!css_tryget(&memcg->css)))
2735 goto retry;
2736 rcu_read_unlock();
2737
2738 return memcg;
2739}
2740
2741#ifdef CONFIG_MEMCG_KMEM
2783#if defined(CONFIG_MEMCG_KMEM) || defined(CONFIG_MMU)
2784static void cancel_charge(struct mem_cgroup *memcg, unsigned int nr_pages)
2785{
2786 if (mem_cgroup_is_root(memcg))
2787 return;
2788
2789 page_counter_uncharge(&memcg->memory, nr_pages);
2790 if (do_memsw_account())

--- 25 unchanged lines hidden (view full) ---

2816 if (unlikely(!css_tryget(&memcg->css)))
2817 goto retry;
2818 rcu_read_unlock();
2819
2820 return memcg;
2821}
2822
2823#ifdef CONFIG_MEMCG_KMEM
2824/*
2825 * The allocated objcg pointers array is not accounted directly.
2826 * Moreover, it should not come from DMA buffer and is not readily
2827 * reclaimable. So those GFP bits should be masked off.
2828 */
2829#define OBJCGS_CLEAR_MASK (__GFP_DMA | __GFP_RECLAIMABLE | __GFP_ACCOUNT)
2830
2742int memcg_alloc_page_obj_cgroups(struct page *page, struct kmem_cache *s,
2743 gfp_t gfp, bool new_page)
2744{
2745 unsigned int objects = objs_per_slab_page(s, page);
2746 unsigned long memcg_data;
2747 void *vec;
2748
2831int memcg_alloc_page_obj_cgroups(struct page *page, struct kmem_cache *s,
2832 gfp_t gfp, bool new_page)
2833{
2834 unsigned int objects = objs_per_slab_page(s, page);
2835 unsigned long memcg_data;
2836 void *vec;
2837
2838 gfp &= ~OBJCGS_CLEAR_MASK;
2749 vec = kcalloc_node(objects, sizeof(struct obj_cgroup *), gfp,
2750 page_to_nid(page));
2751 if (!vec)
2752 return -ENOMEM;
2753
2754 memcg_data = (unsigned long) vec | MEMCG_DATA_OBJCGS;
2755 if (new_page) {
2756 /*

--- 163 unchanged lines hidden (view full) ---

2920 unsigned int nr_pages)
2921{
2922 struct page_counter *counter;
2923 struct mem_cgroup *memcg;
2924 int ret;
2925
2926 memcg = get_mem_cgroup_from_objcg(objcg);
2927
2839 vec = kcalloc_node(objects, sizeof(struct obj_cgroup *), gfp,
2840 page_to_nid(page));
2841 if (!vec)
2842 return -ENOMEM;
2843
2844 memcg_data = (unsigned long) vec | MEMCG_DATA_OBJCGS;
2845 if (new_page) {
2846 /*

--- 163 unchanged lines hidden (view full) ---

3010 unsigned int nr_pages)
3011{
3012 struct page_counter *counter;
3013 struct mem_cgroup *memcg;
3014 int ret;
3015
3016 memcg = get_mem_cgroup_from_objcg(objcg);
3017
2928 ret = try_charge(memcg, gfp, nr_pages);
3018 ret = try_charge_memcg(memcg, gfp, nr_pages);
2929 if (ret)
2930 goto out;
2931
2932 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) &&
2933 !page_counter_try_charge(&memcg->kmem, nr_pages, &counter)) {
2934
2935 /*
2936 * Enforce __GFP_NOFAIL allocation because callers are not

--- 53 unchanged lines hidden (view full) ---

2990 return;
2991
2992 objcg = __page_objcg(page);
2993 obj_cgroup_uncharge_pages(objcg, nr_pages);
2994 page->memcg_data = 0;
2995 obj_cgroup_put(objcg);
2996}
2997
3019 if (ret)
3020 goto out;
3021
3022 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) &&
3023 !page_counter_try_charge(&memcg->kmem, nr_pages, &counter)) {
3024
3025 /*
3026 * Enforce __GFP_NOFAIL allocation because callers are not

--- 53 unchanged lines hidden (view full) ---

3080 return;
3081
3082 objcg = __page_objcg(page);
3083 obj_cgroup_uncharge_pages(objcg, nr_pages);
3084 page->memcg_data = 0;
3085 obj_cgroup_put(objcg);
3086}
3087
3088void mod_objcg_state(struct obj_cgroup *objcg, struct pglist_data *pgdat,
3089 enum node_stat_item idx, int nr)
3090{
3091 unsigned long flags;
3092 struct obj_stock *stock = get_obj_stock(&flags);
3093 int *bytes;
3094
3095 /*
3096 * Save vmstat data in stock and skip vmstat array update unless
3097 * accumulating over a page of vmstat data or when pgdat or idx
3098 * changes.
3099 */
3100 if (stock->cached_objcg != objcg) {
3101 drain_obj_stock(stock);
3102 obj_cgroup_get(objcg);
3103 stock->nr_bytes = atomic_read(&objcg->nr_charged_bytes)
3104 ? atomic_xchg(&objcg->nr_charged_bytes, 0) : 0;
3105 stock->cached_objcg = objcg;
3106 stock->cached_pgdat = pgdat;
3107 } else if (stock->cached_pgdat != pgdat) {
3108 /* Flush the existing cached vmstat data */
3109 if (stock->nr_slab_reclaimable_b) {
3110 mod_objcg_mlstate(objcg, pgdat, NR_SLAB_RECLAIMABLE_B,
3111 stock->nr_slab_reclaimable_b);
3112 stock->nr_slab_reclaimable_b = 0;
3113 }
3114 if (stock->nr_slab_unreclaimable_b) {
3115 mod_objcg_mlstate(objcg, pgdat, NR_SLAB_UNRECLAIMABLE_B,
3116 stock->nr_slab_unreclaimable_b);
3117 stock->nr_slab_unreclaimable_b = 0;
3118 }
3119 stock->cached_pgdat = pgdat;
3120 }
3121
3122 bytes = (idx == NR_SLAB_RECLAIMABLE_B) ? &stock->nr_slab_reclaimable_b
3123 : &stock->nr_slab_unreclaimable_b;
3124 /*
3125 * Even for large object >= PAGE_SIZE, the vmstat data will still be
3126 * cached locally at least once before pushing it out.
3127 */
3128 if (!*bytes) {
3129 *bytes = nr;
3130 nr = 0;
3131 } else {
3132 *bytes += nr;
3133 if (abs(*bytes) > PAGE_SIZE) {
3134 nr = *bytes;
3135 *bytes = 0;
3136 } else {
3137 nr = 0;
3138 }
3139 }
3140 if (nr)
3141 mod_objcg_mlstate(objcg, pgdat, idx, nr);
3142
3143 put_obj_stock(flags);
3144}
3145
2998static bool consume_obj_stock(struct obj_cgroup *objcg, unsigned int nr_bytes)
2999{
3146static bool consume_obj_stock(struct obj_cgroup *objcg, unsigned int nr_bytes)
3147{
3000 struct memcg_stock_pcp *stock;
3001 unsigned long flags;
3148 unsigned long flags;
3149 struct obj_stock *stock = get_obj_stock(&flags);
3002 bool ret = false;
3003
3150 bool ret = false;
3151
3004 local_irq_save(flags);
3005
3006 stock = this_cpu_ptr(&memcg_stock);
3007 if (objcg == stock->cached_objcg && stock->nr_bytes >= nr_bytes) {
3008 stock->nr_bytes -= nr_bytes;
3009 ret = true;
3010 }
3011
3152 if (objcg == stock->cached_objcg && stock->nr_bytes >= nr_bytes) {
3153 stock->nr_bytes -= nr_bytes;
3154 ret = true;
3155 }
3156
3012 local_irq_restore(flags);
3157 put_obj_stock(flags);
3013
3014 return ret;
3015}
3016
3158
3159 return ret;
3160}
3161
3017static void drain_obj_stock(struct memcg_stock_pcp *stock)
3162static void drain_obj_stock(struct obj_stock *stock)
3018{
3019 struct obj_cgroup *old = stock->cached_objcg;
3020
3021 if (!old)
3022 return;
3023
3024 if (stock->nr_bytes) {
3025 unsigned int nr_pages = stock->nr_bytes >> PAGE_SHIFT;

--- 11 unchanged lines hidden (view full) ---

3037 * How often it's flushed is a trade-off between the memory
3038 * limit enforcement accuracy and potential CPU contention,
3039 * so it might be changed in the future.
3040 */
3041 atomic_add(nr_bytes, &old->nr_charged_bytes);
3042 stock->nr_bytes = 0;
3043 }
3044
3163{
3164 struct obj_cgroup *old = stock->cached_objcg;
3165
3166 if (!old)
3167 return;
3168
3169 if (stock->nr_bytes) {
3170 unsigned int nr_pages = stock->nr_bytes >> PAGE_SHIFT;

--- 11 unchanged lines hidden (view full) ---

3182 * How often it's flushed is a trade-off between the memory
3183 * limit enforcement accuracy and potential CPU contention,
3184 * so it might be changed in the future.
3185 */
3186 atomic_add(nr_bytes, &old->nr_charged_bytes);
3187 stock->nr_bytes = 0;
3188 }
3189
3190 /*
3191 * Flush the vmstat data in current stock
3192 */
3193 if (stock->nr_slab_reclaimable_b || stock->nr_slab_unreclaimable_b) {
3194 if (stock->nr_slab_reclaimable_b) {
3195 mod_objcg_mlstate(old, stock->cached_pgdat,
3196 NR_SLAB_RECLAIMABLE_B,
3197 stock->nr_slab_reclaimable_b);
3198 stock->nr_slab_reclaimable_b = 0;
3199 }
3200 if (stock->nr_slab_unreclaimable_b) {
3201 mod_objcg_mlstate(old, stock->cached_pgdat,
3202 NR_SLAB_UNRECLAIMABLE_B,
3203 stock->nr_slab_unreclaimable_b);
3204 stock->nr_slab_unreclaimable_b = 0;
3205 }
3206 stock->cached_pgdat = NULL;
3207 }
3208
3045 obj_cgroup_put(old);
3046 stock->cached_objcg = NULL;
3047}
3048
3049static bool obj_stock_flush_required(struct memcg_stock_pcp *stock,
3050 struct mem_cgroup *root_memcg)
3051{
3052 struct mem_cgroup *memcg;
3053
3209 obj_cgroup_put(old);
3210 stock->cached_objcg = NULL;
3211}
3212
3213static bool obj_stock_flush_required(struct memcg_stock_pcp *stock,
3214 struct mem_cgroup *root_memcg)
3215{
3216 struct mem_cgroup *memcg;
3217
3054 if (stock->cached_objcg) {
3055 memcg = obj_cgroup_memcg(stock->cached_objcg);
3218 if (in_task() && stock->task_obj.cached_objcg) {
3219 memcg = obj_cgroup_memcg(stock->task_obj.cached_objcg);
3056 if (memcg && mem_cgroup_is_descendant(memcg, root_memcg))
3057 return true;
3058 }
3220 if (memcg && mem_cgroup_is_descendant(memcg, root_memcg))
3221 return true;
3222 }
3223 if (stock->irq_obj.cached_objcg) {
3224 memcg = obj_cgroup_memcg(stock->irq_obj.cached_objcg);
3225 if (memcg && mem_cgroup_is_descendant(memcg, root_memcg))
3226 return true;
3227 }
3059
3060 return false;
3061}
3062
3228
3229 return false;
3230}
3231
3063static void refill_obj_stock(struct obj_cgroup *objcg, unsigned int nr_bytes)
3232static void refill_obj_stock(struct obj_cgroup *objcg, unsigned int nr_bytes,
3233 bool allow_uncharge)
3064{
3234{
3065 struct memcg_stock_pcp *stock;
3066 unsigned long flags;
3235 unsigned long flags;
3236 struct obj_stock *stock = get_obj_stock(&flags);
3237 unsigned int nr_pages = 0;
3067
3238
3068 local_irq_save(flags);
3069
3070 stock = this_cpu_ptr(&memcg_stock);
3071 if (stock->cached_objcg != objcg) { /* reset if necessary */
3072 drain_obj_stock(stock);
3073 obj_cgroup_get(objcg);
3074 stock->cached_objcg = objcg;
3239 if (stock->cached_objcg != objcg) { /* reset if necessary */
3240 drain_obj_stock(stock);
3241 obj_cgroup_get(objcg);
3242 stock->cached_objcg = objcg;
3075 stock->nr_bytes = atomic_xchg(&objcg->nr_charged_bytes, 0);
3243 stock->nr_bytes = atomic_read(&objcg->nr_charged_bytes)
3244 ? atomic_xchg(&objcg->nr_charged_bytes, 0) : 0;
3245 allow_uncharge = true; /* Allow uncharge when objcg changes */
3076 }
3077 stock->nr_bytes += nr_bytes;
3078
3246 }
3247 stock->nr_bytes += nr_bytes;
3248
3079 if (stock->nr_bytes > PAGE_SIZE)
3080 drain_obj_stock(stock);
3249 if (allow_uncharge && (stock->nr_bytes > PAGE_SIZE)) {
3250 nr_pages = stock->nr_bytes >> PAGE_SHIFT;
3251 stock->nr_bytes &= (PAGE_SIZE - 1);
3252 }
3081
3253
3082 local_irq_restore(flags);
3254 put_obj_stock(flags);
3255
3256 if (nr_pages)
3257 obj_cgroup_uncharge_pages(objcg, nr_pages);
3083}
3084
3085int obj_cgroup_charge(struct obj_cgroup *objcg, gfp_t gfp, size_t size)
3086{
3087 unsigned int nr_pages, nr_bytes;
3088 int ret;
3089
3090 if (consume_obj_stock(objcg, size))
3091 return 0;
3092
3093 /*
3258}
3259
3260int obj_cgroup_charge(struct obj_cgroup *objcg, gfp_t gfp, size_t size)
3261{
3262 unsigned int nr_pages, nr_bytes;
3263 int ret;
3264
3265 if (consume_obj_stock(objcg, size))
3266 return 0;
3267
3268 /*
3094 * In theory, memcg->nr_charged_bytes can have enough
3269 * In theory, objcg->nr_charged_bytes can have enough
3095 * pre-charged bytes to satisfy the allocation. However,
3270 * pre-charged bytes to satisfy the allocation. However,
3096 * flushing memcg->nr_charged_bytes requires two atomic
3097 * operations, and memcg->nr_charged_bytes can't be big,
3098 * so it's better to ignore it and try grab some new pages.
3099 * memcg->nr_charged_bytes will be flushed in
3100 * refill_obj_stock(), called from this function or
3101 * independently later.
3271 * flushing objcg->nr_charged_bytes requires two atomic
3272 * operations, and objcg->nr_charged_bytes can't be big.
3273 * The shared objcg->nr_charged_bytes can also become a
3274 * performance bottleneck if all tasks of the same memcg are
3275 * trying to update it. So it's better to ignore it and try
3276 * grab some new pages. The stock's nr_bytes will be flushed to
3277 * objcg->nr_charged_bytes later on when objcg changes.
3278 *
3279 * The stock's nr_bytes may contain enough pre-charged bytes
3280 * to allow one less page from being charged, but we can't rely
3281 * on the pre-charged bytes not being changed outside of
3282 * consume_obj_stock() or refill_obj_stock(). So ignore those
3283 * pre-charged bytes as well when charging pages. To avoid a
3284 * page uncharge right after a page charge, we set the
3285 * allow_uncharge flag to false when calling refill_obj_stock()
3286 * to temporarily allow the pre-charged bytes to exceed the page
3287 * size limit. The maximum reachable value of the pre-charged
3288 * bytes is (sizeof(object) + PAGE_SIZE - 2) if there is no data
3289 * race.
3102 */
3103 nr_pages = size >> PAGE_SHIFT;
3104 nr_bytes = size & (PAGE_SIZE - 1);
3105
3106 if (nr_bytes)
3107 nr_pages += 1;
3108
3109 ret = obj_cgroup_charge_pages(objcg, gfp, nr_pages);
3110 if (!ret && nr_bytes)
3290 */
3291 nr_pages = size >> PAGE_SHIFT;
3292 nr_bytes = size & (PAGE_SIZE - 1);
3293
3294 if (nr_bytes)
3295 nr_pages += 1;
3296
3297 ret = obj_cgroup_charge_pages(objcg, gfp, nr_pages);
3298 if (!ret && nr_bytes)
3111 refill_obj_stock(objcg, PAGE_SIZE - nr_bytes);
3299 refill_obj_stock(objcg, PAGE_SIZE - nr_bytes, false);
3112
3113 return ret;
3114}
3115
3116void obj_cgroup_uncharge(struct obj_cgroup *objcg, size_t size)
3117{
3300
3301 return ret;
3302}
3303
3304void obj_cgroup_uncharge(struct obj_cgroup *objcg, size_t size)
3305{
3118 refill_obj_stock(objcg, size);
3306 refill_obj_stock(objcg, size, true);
3119}
3120
3121#endif /* CONFIG_MEMCG_KMEM */
3122
3123/*
3124 * Because page_memcg(head) is not set on tails, set it now.
3125 */
3126void split_page_memcg(struct page *head, unsigned int nr)

--- 2217 unchanged lines hidden (view full) ---

5344 return NULL;
5345
5346 /*
5347 * Handle MEMORY_DEVICE_PRIVATE which are ZONE_DEVICE page belonging to
5348 * a device and because they are not accessible by CPU they are store
5349 * as special swap entry in the CPU page table.
5350 */
5351 if (is_device_private_entry(ent)) {
3307}
3308
3309#endif /* CONFIG_MEMCG_KMEM */
3310
3311/*
3312 * Because page_memcg(head) is not set on tails, set it now.
3313 */
3314void split_page_memcg(struct page *head, unsigned int nr)

--- 2217 unchanged lines hidden (view full) ---

5532 return NULL;
5533
5534 /*
5535 * Handle MEMORY_DEVICE_PRIVATE which are ZONE_DEVICE page belonging to
5536 * a device and because they are not accessible by CPU they are store
5537 * as special swap entry in the CPU page table.
5538 */
5539 if (is_device_private_entry(ent)) {
5352 page = device_private_entry_to_page(ent);
5540 page = pfn_swap_entry_to_page(ent);
5353 /*
5354 * MEMORY_DEVICE_PRIVATE means ZONE_DEVICE page and which have
5355 * a refcount of 1 when free (unlike normal page)
5356 */
5357 if (!page_ref_add_unless(page, 1, 1))
5358 return NULL;
5359 return page;
5360 }

--- 1090 unchanged lines hidden (view full) ---

6451
6452 ep += unclaimed;
6453 }
6454
6455 return ep;
6456}
6457
6458/**
5541 /*
5542 * MEMORY_DEVICE_PRIVATE means ZONE_DEVICE page and which have
5543 * a refcount of 1 when free (unlike normal page)
5544 */
5545 if (!page_ref_add_unless(page, 1, 1))
5546 return NULL;
5547 return page;
5548 }

--- 1090 unchanged lines hidden (view full) ---

6639
6640 ep += unclaimed;
6641 }
6642
6643 return ep;
6644}
6645
6646/**
6459 * mem_cgroup_protected - check if memory consumption is in the normal range
6647 * mem_cgroup_calculate_protection - check if memory consumption is in the normal range
6460 * @root: the top ancestor of the sub-tree being checked
6461 * @memcg: the memory cgroup to check
6462 *
6463 * WARNING: This function is not stateless! It can only be used as part
6464 * of a top-down tree iteration, not for isolated queries.
6465 */
6466void mem_cgroup_calculate_protection(struct mem_cgroup *root,
6467 struct mem_cgroup *memcg)

--- 68 unchanged lines hidden (view full) ---

6536
6537/**
6538 * mem_cgroup_charge - charge a newly allocated page to a cgroup
6539 * @page: page to charge
6540 * @mm: mm context of the victim
6541 * @gfp_mask: reclaim mode
6542 *
6543 * Try to charge @page to the memcg that @mm belongs to, reclaiming
6648 * @root: the top ancestor of the sub-tree being checked
6649 * @memcg: the memory cgroup to check
6650 *
6651 * WARNING: This function is not stateless! It can only be used as part
6652 * of a top-down tree iteration, not for isolated queries.
6653 */
6654void mem_cgroup_calculate_protection(struct mem_cgroup *root,
6655 struct mem_cgroup *memcg)

--- 68 unchanged lines hidden (view full) ---

6724
6725/**
6726 * mem_cgroup_charge - charge a newly allocated page to a cgroup
6727 * @page: page to charge
6728 * @mm: mm context of the victim
6729 * @gfp_mask: reclaim mode
6730 *
6731 * Try to charge @page to the memcg that @mm belongs to, reclaiming
6544 * pages according to @gfp_mask if necessary.
6732 * pages according to @gfp_mask if necessary. if @mm is NULL, try to
6733 * charge to the active memcg.
6545 *
6546 * Do not use this for pages allocated for swapin.
6547 *
6548 * Returns 0 on success. Otherwise, an error code is returned.
6549 */
6550int mem_cgroup_charge(struct page *page, struct mm_struct *mm, gfp_t gfp_mask)
6551{
6552 struct mem_cgroup *memcg;

--- 113 unchanged lines hidden (view full) ---

6666 css_put(&ug->memcg->css);
6667}
6668
6669static void uncharge_page(struct page *page, struct uncharge_gather *ug)
6670{
6671 unsigned long nr_pages;
6672 struct mem_cgroup *memcg;
6673 struct obj_cgroup *objcg;
6734 *
6735 * Do not use this for pages allocated for swapin.
6736 *
6737 * Returns 0 on success. Otherwise, an error code is returned.
6738 */
6739int mem_cgroup_charge(struct page *page, struct mm_struct *mm, gfp_t gfp_mask)
6740{
6741 struct mem_cgroup *memcg;

--- 113 unchanged lines hidden (view full) ---

6855 css_put(&ug->memcg->css);
6856}
6857
6858static void uncharge_page(struct page *page, struct uncharge_gather *ug)
6859{
6860 unsigned long nr_pages;
6861 struct mem_cgroup *memcg;
6862 struct obj_cgroup *objcg;
6863 bool use_objcg = PageMemcgKmem(page);
6674
6675 VM_BUG_ON_PAGE(PageLRU(page), page);
6676
6677 /*
6678 * Nobody should be changing or seriously looking at
6679 * page memcg or objcg at this point, we have fully
6680 * exclusive access to the page.
6681 */
6864
6865 VM_BUG_ON_PAGE(PageLRU(page), page);
6866
6867 /*
6868 * Nobody should be changing or seriously looking at
6869 * page memcg or objcg at this point, we have fully
6870 * exclusive access to the page.
6871 */
6682 if (PageMemcgKmem(page)) {
6872 if (use_objcg) {
6683 objcg = __page_objcg(page);
6684 /*
6685 * This get matches the put at the end of the function and
6686 * kmem pages do not hold memcg references anymore.
6687 */
6688 memcg = get_mem_cgroup_from_objcg(objcg);
6689 } else {
6690 memcg = __page_memcg(page);

--- 11 unchanged lines hidden (view full) ---

6702 ug->dummy_page = page;
6703
6704 /* pairs with css_put in uncharge_batch */
6705 css_get(&memcg->css);
6706 }
6707
6708 nr_pages = compound_nr(page);
6709
6873 objcg = __page_objcg(page);
6874 /*
6875 * This get matches the put at the end of the function and
6876 * kmem pages do not hold memcg references anymore.
6877 */
6878 memcg = get_mem_cgroup_from_objcg(objcg);
6879 } else {
6880 memcg = __page_memcg(page);

--- 11 unchanged lines hidden (view full) ---

6892 ug->dummy_page = page;
6893
6894 /* pairs with css_put in uncharge_batch */
6895 css_get(&memcg->css);
6896 }
6897
6898 nr_pages = compound_nr(page);
6899
6710 if (PageMemcgKmem(page)) {
6900 if (use_objcg) {
6711 ug->nr_memory += nr_pages;
6712 ug->nr_kmem += nr_pages;
6713
6714 page->memcg_data = 0;
6715 obj_cgroup_put(objcg);
6716 } else {
6717 /* LRU pages aren't accounted at the root level */
6718 if (!mem_cgroup_is_root(memcg))

--- 82 unchanged lines hidden (view full) ---

6801 memcg = page_memcg(oldpage);
6802 VM_WARN_ON_ONCE_PAGE(!memcg, oldpage);
6803 if (!memcg)
6804 return;
6805
6806 /* Force-charge the new page. The old one will be freed soon */
6807 nr_pages = thp_nr_pages(newpage);
6808
6901 ug->nr_memory += nr_pages;
6902 ug->nr_kmem += nr_pages;
6903
6904 page->memcg_data = 0;
6905 obj_cgroup_put(objcg);
6906 } else {
6907 /* LRU pages aren't accounted at the root level */
6908 if (!mem_cgroup_is_root(memcg))

--- 82 unchanged lines hidden (view full) ---

6991 memcg = page_memcg(oldpage);
6992 VM_WARN_ON_ONCE_PAGE(!memcg, oldpage);
6993 if (!memcg)
6994 return;
6995
6996 /* Force-charge the new page. The old one will be freed soon */
6997 nr_pages = thp_nr_pages(newpage);
6998
6809 page_counter_charge(&memcg->memory, nr_pages);
6810 if (do_memsw_account())
6811 page_counter_charge(&memcg->memsw, nr_pages);
6999 if (!mem_cgroup_is_root(memcg)) {
7000 page_counter_charge(&memcg->memory, nr_pages);
7001 if (do_memsw_account())
7002 page_counter_charge(&memcg->memsw, nr_pages);
7003 }
6812
6813 css_get(&memcg->css);
6814 commit_charge(newpage, memcg);
6815
6816 local_irq_save(flags);
6817 mem_cgroup_charge_statistics(memcg, newpage, nr_pages);
6818 memcg_check_events(memcg, newpage);
6819 local_irq_restore(flags);

--- 507 unchanged lines hidden --- 		7004
7005 css_get(&memcg->css);
7006 commit_charge(newpage, memcg);
7007
7008 local_irq_save(flags);
7009 mem_cgroup_charge_statistics(memcg, newpage, nr_pages);
7010 memcg_check_events(memcg, newpage);
7011 local_irq_restore(flags);

--- 507 unchanged lines hidden ---