diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 78538d565edd6ccf679db53183fbe58215b1086a..9e7276c63a82db493af5ad7c3803da354d50fe85 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -68,6 +68,7 @@ #include #include "slab.h" #include +#include #include #ifdef CONFIG_TEXT_UNEVICTABLE @@ -112,6 +113,13 @@ static DECLARE_WAIT_QUEUE_HEAD(memcg_cgwb_frn_waitq); static struct workqueue_struct *memcg_wmark_wq; +struct event_lock { + local_lock_t l; +}; +static DEFINE_PER_CPU(struct event_lock, event_lock) = { + .l = INIT_LOCAL_LOCK(l), +}; + /* Whether legacy memory+swap accounting is active */ static bool do_memsw_account(void) { @@ -2493,6 +2501,7 @@ void unlock_page_memcg(struct page *page) EXPORT_SYMBOL(unlock_page_memcg); struct memcg_stock_pcp { + local_lock_t lock; struct mem_cgroup *cached; /* this never be root cgroup */ unsigned int nr_pages; @@ -2544,7 +2553,7 @@ static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages) if (nr_pages > MEMCG_CHARGE_BATCH) return ret; - local_irq_save(flags); + local_lock_irqsave(&memcg_stock.lock, flags); stock = this_cpu_ptr(&memcg_stock); if (memcg == stock->cached && stock->nr_pages >= nr_pages) { @@ -2552,7 +2561,7 @@ static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages) ret = true; } - local_irq_restore(flags); + local_unlock_irqrestore(&memcg_stock.lock, flags); return ret; } @@ -2587,14 +2596,14 @@ static void drain_local_stock(struct work_struct *dummy) * The only protection from memory hotplug vs. drain_stock races is * that we always operate on local CPU stock here with IRQ disabled */ - local_irq_save(flags); + local_lock_irqsave(&memcg_stock.lock, flags); stock = this_cpu_ptr(&memcg_stock); drain_obj_stock(stock); drain_stock(stock); clear_bit(FLUSHING_CACHED_CHARGE, &stock->flags); - local_irq_restore(flags); + local_unlock_irqrestore(&memcg_stock.lock, flags); } /* @@ -2606,7 +2615,7 @@ static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages) struct memcg_stock_pcp *stock; unsigned long flags; - local_irq_save(flags); + local_lock_irqsave(&memcg_stock.lock, flags); stock = this_cpu_ptr(&memcg_stock); if (stock->cached != memcg) { /* reset if necessary */ @@ -2619,7 +2628,7 @@ static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages) if (stock->nr_pages > MEMCG_CHARGE_BATCH) drain_stock(stock); - local_irq_restore(flags); + local_unlock_irqrestore(&memcg_stock.lock, flags); } /* @@ -2639,7 +2648,7 @@ void drain_all_stock(struct mem_cgroup *root_memcg) * as well as workers from this path always operate on the local * per-cpu data. CPU up doesn't touch memcg_stock at all. */ - curcpu = get_cpu(); + curcpu = get_cpu_light(); for_each_online_cpu(cpu) { struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu); struct mem_cgroup *memcg; @@ -2662,7 +2671,7 @@ void drain_all_stock(struct mem_cgroup *root_memcg) schedule_work_on(cpu, &stock->work); } } - put_cpu(); + put_cpu_light(); mutex_unlock(&percpu_charge_mutex); } @@ -3522,7 +3531,7 @@ static bool consume_obj_stock(struct obj_cgroup *objcg, unsigned int nr_bytes) unsigned long flags; bool ret = false; - local_irq_save(flags); + local_lock_irqsave(&memcg_stock.lock, flags); stock = this_cpu_ptr(&memcg_stock); if (objcg == stock->cached_objcg && stock->nr_bytes >= nr_bytes) { @@ -3530,7 +3539,7 @@ static bool consume_obj_stock(struct obj_cgroup *objcg, unsigned int nr_bytes) ret = true; } - local_irq_restore(flags); + local_unlock_irqrestore(&memcg_stock.lock, flags); return ret; } @@ -3597,7 +3606,7 @@ static void refill_obj_stock(struct obj_cgroup *objcg, unsigned int nr_bytes) struct memcg_stock_pcp *stock; unsigned long flags; - local_irq_save(flags); + local_lock_irqsave(&memcg_stock.lock, flags); stock = this_cpu_ptr(&memcg_stock); if (stock->cached_objcg != objcg) { /* reset if necessary */ @@ -3611,7 +3620,7 @@ static void refill_obj_stock(struct obj_cgroup *objcg, unsigned int nr_bytes) if (stock->nr_bytes > PAGE_SIZE) drain_obj_stock(stock); - local_irq_restore(flags); + local_unlock_irqrestore(&memcg_stock.lock, flags); } int obj_cgroup_charge(struct obj_cgroup *objcg, gfp_t gfp, size_t size) @@ -7478,12 +7487,12 @@ static int mem_cgroup_move_account(struct page *page, kidled_mem_cgroup_move_stats(from, to, page, nr_pages << PAGE_SHIFT); - local_irq_disable(); + local_lock_irq(&event_lock.l); mem_cgroup_charge_statistics(to, page, nr_pages); memcg_check_events(to, page); mem_cgroup_charge_statistics(from, page, -nr_pages); memcg_check_events(from, page); - local_irq_enable(); + local_unlock_irq(&event_lock.l); out_unlock: unlock_page(page); out: @@ -8640,10 +8649,10 @@ int mem_cgroup_charge(struct page *page, struct mm_struct *mm, gfp_t gfp_mask) css_get(&memcg->css); commit_charge(page, memcg); - local_irq_disable(); + local_lock_irq(&event_lock.l); mem_cgroup_charge_statistics(memcg, page, nr_pages); memcg_check_events(memcg, page); - local_irq_enable(); + local_unlock_irq(&event_lock.l); /* * Cgroup1's unified memory+swap counter has been charged with the @@ -8699,11 +8708,11 @@ static void uncharge_batch(const struct uncharge_gather *ug) memcg_oom_recover(ug->memcg); } - local_irq_save(flags); + local_lock_irqsave(&event_lock.l, flags); __count_memcg_events(ug->memcg, PGPGOUT, ug->pgpgout); __this_cpu_add(ug->memcg->vmstats_percpu->nr_page_events, ug->nr_pages); memcg_check_events(ug->memcg, ug->dummy_page); - local_irq_restore(flags); + local_unlock_irqrestore(&event_lock.l, flags); /* drop reference from uncharge_page */ css_put(&ug->memcg->css); @@ -8857,10 +8866,10 @@ void mem_cgroup_migrate(struct page *oldpage, struct page *newpage) css_get(&memcg->css); commit_charge(newpage, memcg); - local_irq_save(flags); + local_lock_irqsave(&event_lock.l, flags); mem_cgroup_charge_statistics(memcg, newpage, nr_pages); memcg_check_events(memcg, newpage); - local_irq_restore(flags); + local_unlock_irqrestore(&event_lock.l, flags); } DEFINE_STATIC_KEY_FALSE(memcg_sockets_enabled_key); @@ -9064,9 +9073,13 @@ static int __init mem_cgroup_init(void) cpuhp_setup_state_nocalls(CPUHP_MM_MEMCQ_DEAD, "mm/memctrl:dead", NULL, memcg_hotplug_cpu_dead); - for_each_possible_cpu(cpu) - INIT_WORK(&per_cpu_ptr(&memcg_stock, cpu)->work, - drain_local_stock); + for_each_possible_cpu(cpu) { + struct memcg_stock_pcp *stock; + + stock = per_cpu_ptr(&memcg_stock, cpu); + INIT_WORK(&stock->work, drain_local_stock); + local_lock_init(&stock->lock); + } for_each_node(node) { struct mem_cgroup_tree_per_node *rtpn; @@ -9115,6 +9128,7 @@ void mem_cgroup_swapout(struct page *page, swp_entry_t entry) struct mem_cgroup *memcg, *swap_memcg; unsigned int nr_entries; unsigned short oldid; + unsigned long flags; VM_BUG_ON_PAGE(PageLRU(page), page); VM_BUG_ON_PAGE(page_count(page), page); @@ -9160,9 +9174,13 @@ void mem_cgroup_swapout(struct page *page, swp_entry_t entry) * important here to have the interrupts disabled because it is the * only synchronisation we have for updating the per-CPU variables. */ + local_lock_irqsave(&event_lock.l, flags); +#ifndef CONFIG_PREEMPT_RT VM_BUG_ON(!irqs_disabled()); +#endif mem_cgroup_charge_statistics(memcg, page, -nr_entries); memcg_check_events(memcg, page); + local_unlock_irqrestore(&event_lock.l, flags); css_put(&memcg->css); }