18b712842SChris Mason /* 28b712842SChris Mason * Copyright (C) 2007 Oracle. All rights reserved. 308a9ff32SQu Wenruo * Copyright (C) 2014 Fujitsu. All rights reserved. 48b712842SChris Mason * 58b712842SChris Mason * This program is free software; you can redistribute it and/or 68b712842SChris Mason * modify it under the terms of the GNU General Public 78b712842SChris Mason * License v2 as published by the Free Software Foundation. 88b712842SChris Mason * 98b712842SChris Mason * This program is distributed in the hope that it will be useful, 108b712842SChris Mason * but WITHOUT ANY WARRANTY; without even the implied warranty of 118b712842SChris Mason * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 128b712842SChris Mason * General Public License for more details. 138b712842SChris Mason * 148b712842SChris Mason * You should have received a copy of the GNU General Public 158b712842SChris Mason * License along with this program; if not, write to the 168b712842SChris Mason * Free Software Foundation, Inc., 59 Temple Place - Suite 330, 178b712842SChris Mason * Boston, MA 021110-1307, USA. 188b712842SChris Mason */ 198b712842SChris Mason 208b712842SChris Mason #include <linux/kthread.h> 215a0e3ad6STejun Heo #include <linux/slab.h> 228b712842SChris Mason #include <linux/list.h> 238b712842SChris Mason #include <linux/spinlock.h> 248b712842SChris Mason #include <linux/freezer.h> 2508a9ff32SQu Wenruo #include <linux/workqueue.h> 268b712842SChris Mason #include "async-thread.h" 2752483bc2SQu Wenruo #include "ctree.h" 288b712842SChris Mason 29a046e9c8SQu Wenruo #define WORK_DONE_BIT 0 30a046e9c8SQu Wenruo #define WORK_ORDER_DONE_BIT 1 31a046e9c8SQu Wenruo #define WORK_HIGH_PRIO_BIT 2 324a69a410SChris Mason 330bd9289cSQu Wenruo #define NO_THRESHOLD (-1) 340bd9289cSQu Wenruo #define DFT_THRESHOLD (32) 350bd9289cSQu Wenruo 36d458b054SQu Wenruo struct __btrfs_workqueue { 3708a9ff32SQu Wenruo struct workqueue_struct *normal_wq; 3808a9ff32SQu Wenruo /* List head pointing to ordered work list */ 3908a9ff32SQu Wenruo struct list_head ordered_list; 4008a9ff32SQu Wenruo 4108a9ff32SQu Wenruo /* Spinlock for ordered_list */ 4208a9ff32SQu Wenruo spinlock_t list_lock; 430bd9289cSQu Wenruo 440bd9289cSQu Wenruo /* Thresholding related variants */ 450bd9289cSQu Wenruo atomic_t pending; 460bd9289cSQu Wenruo int max_active; 470bd9289cSQu Wenruo int current_max; 480bd9289cSQu Wenruo int thresh; 490bd9289cSQu Wenruo unsigned int count; 500bd9289cSQu Wenruo spinlock_t thres_lock; 5108a9ff32SQu Wenruo }; 5208a9ff32SQu Wenruo 53d458b054SQu Wenruo struct btrfs_workqueue { 54d458b054SQu Wenruo struct __btrfs_workqueue *normal; 55d458b054SQu Wenruo struct __btrfs_workqueue *high; 561ca08976SQu Wenruo }; 571ca08976SQu Wenruo 58d458b054SQu Wenruo static inline struct __btrfs_workqueue 59c3a46891SQu Wenruo *__btrfs_alloc_workqueue(const char *name, int flags, int max_active, 60c3a46891SQu Wenruo int thresh) 6108a9ff32SQu Wenruo { 62d458b054SQu Wenruo struct __btrfs_workqueue *ret = kzalloc(sizeof(*ret), GFP_NOFS); 6308a9ff32SQu Wenruo 6408a9ff32SQu Wenruo if (unlikely(!ret)) 6508a9ff32SQu Wenruo return NULL; 6608a9ff32SQu Wenruo 670bd9289cSQu Wenruo ret->max_active = max_active; 680bd9289cSQu Wenruo atomic_set(&ret->pending, 0); 690bd9289cSQu Wenruo if (thresh == 0) 700bd9289cSQu Wenruo thresh = DFT_THRESHOLD; 710bd9289cSQu Wenruo /* For low threshold, disabling threshold is a better choice */ 720bd9289cSQu Wenruo if (thresh < DFT_THRESHOLD) { 730bd9289cSQu Wenruo ret->current_max = max_active; 740bd9289cSQu Wenruo ret->thresh = NO_THRESHOLD; 750bd9289cSQu Wenruo } else { 760bd9289cSQu Wenruo ret->current_max = 1; 770bd9289cSQu Wenruo ret->thresh = thresh; 780bd9289cSQu Wenruo } 790bd9289cSQu Wenruo 801ca08976SQu Wenruo if (flags & WQ_HIGHPRI) 811ca08976SQu Wenruo ret->normal_wq = alloc_workqueue("%s-%s-high", flags, 820bd9289cSQu Wenruo ret->max_active, 830bd9289cSQu Wenruo "btrfs", name); 841ca08976SQu Wenruo else 851ca08976SQu Wenruo ret->normal_wq = alloc_workqueue("%s-%s", flags, 860bd9289cSQu Wenruo ret->max_active, "btrfs", 870bd9289cSQu Wenruo name); 8808a9ff32SQu Wenruo if (unlikely(!ret->normal_wq)) { 8908a9ff32SQu Wenruo kfree(ret); 9008a9ff32SQu Wenruo return NULL; 9108a9ff32SQu Wenruo } 9208a9ff32SQu Wenruo 9308a9ff32SQu Wenruo INIT_LIST_HEAD(&ret->ordered_list); 9408a9ff32SQu Wenruo spin_lock_init(&ret->list_lock); 950bd9289cSQu Wenruo spin_lock_init(&ret->thres_lock); 96c3a46891SQu Wenruo trace_btrfs_workqueue_alloc(ret, name, flags & WQ_HIGHPRI); 9708a9ff32SQu Wenruo return ret; 9808a9ff32SQu Wenruo } 9908a9ff32SQu Wenruo 1001ca08976SQu Wenruo static inline void 101d458b054SQu Wenruo __btrfs_destroy_workqueue(struct __btrfs_workqueue *wq); 1021ca08976SQu Wenruo 103c3a46891SQu Wenruo struct btrfs_workqueue *btrfs_alloc_workqueue(const char *name, 1041ca08976SQu Wenruo int flags, 1050bd9289cSQu Wenruo int max_active, 1060bd9289cSQu Wenruo int thresh) 1071ca08976SQu Wenruo { 108d458b054SQu Wenruo struct btrfs_workqueue *ret = kzalloc(sizeof(*ret), GFP_NOFS); 1091ca08976SQu Wenruo 1101ca08976SQu Wenruo if (unlikely(!ret)) 1111ca08976SQu Wenruo return NULL; 1121ca08976SQu Wenruo 1131ca08976SQu Wenruo ret->normal = __btrfs_alloc_workqueue(name, flags & ~WQ_HIGHPRI, 1140bd9289cSQu Wenruo max_active, thresh); 1151ca08976SQu Wenruo if (unlikely(!ret->normal)) { 1161ca08976SQu Wenruo kfree(ret); 1171ca08976SQu Wenruo return NULL; 1181ca08976SQu Wenruo } 1191ca08976SQu Wenruo 1201ca08976SQu Wenruo if (flags & WQ_HIGHPRI) { 1210bd9289cSQu Wenruo ret->high = __btrfs_alloc_workqueue(name, flags, max_active, 1220bd9289cSQu Wenruo thresh); 1231ca08976SQu Wenruo if (unlikely(!ret->high)) { 1241ca08976SQu Wenruo __btrfs_destroy_workqueue(ret->normal); 1251ca08976SQu Wenruo kfree(ret); 1261ca08976SQu Wenruo return NULL; 1271ca08976SQu Wenruo } 1281ca08976SQu Wenruo } 1291ca08976SQu Wenruo return ret; 1301ca08976SQu Wenruo } 1311ca08976SQu Wenruo 1320bd9289cSQu Wenruo /* 1330bd9289cSQu Wenruo * Hook for threshold which will be called in btrfs_queue_work. 1340bd9289cSQu Wenruo * This hook WILL be called in IRQ handler context, 1350bd9289cSQu Wenruo * so workqueue_set_max_active MUST NOT be called in this hook 1360bd9289cSQu Wenruo */ 137d458b054SQu Wenruo static inline void thresh_queue_hook(struct __btrfs_workqueue *wq) 1380bd9289cSQu Wenruo { 1390bd9289cSQu Wenruo if (wq->thresh == NO_THRESHOLD) 1400bd9289cSQu Wenruo return; 1410bd9289cSQu Wenruo atomic_inc(&wq->pending); 1420bd9289cSQu Wenruo } 1430bd9289cSQu Wenruo 1440bd9289cSQu Wenruo /* 1450bd9289cSQu Wenruo * Hook for threshold which will be called before executing the work, 1460bd9289cSQu Wenruo * This hook is called in kthread content. 1470bd9289cSQu Wenruo * So workqueue_set_max_active is called here. 1480bd9289cSQu Wenruo */ 149d458b054SQu Wenruo static inline void thresh_exec_hook(struct __btrfs_workqueue *wq) 1500bd9289cSQu Wenruo { 1510bd9289cSQu Wenruo int new_max_active; 1520bd9289cSQu Wenruo long pending; 1530bd9289cSQu Wenruo int need_change = 0; 1540bd9289cSQu Wenruo 1550bd9289cSQu Wenruo if (wq->thresh == NO_THRESHOLD) 1560bd9289cSQu Wenruo return; 1570bd9289cSQu Wenruo 1580bd9289cSQu Wenruo atomic_dec(&wq->pending); 1590bd9289cSQu Wenruo spin_lock(&wq->thres_lock); 1600bd9289cSQu Wenruo /* 1610bd9289cSQu Wenruo * Use wq->count to limit the calling frequency of 1620bd9289cSQu Wenruo * workqueue_set_max_active. 1630bd9289cSQu Wenruo */ 1640bd9289cSQu Wenruo wq->count++; 1650bd9289cSQu Wenruo wq->count %= (wq->thresh / 4); 1660bd9289cSQu Wenruo if (!wq->count) 1670bd9289cSQu Wenruo goto out; 1680bd9289cSQu Wenruo new_max_active = wq->current_max; 1690bd9289cSQu Wenruo 1700bd9289cSQu Wenruo /* 1710bd9289cSQu Wenruo * pending may be changed later, but it's OK since we really 1720bd9289cSQu Wenruo * don't need it so accurate to calculate new_max_active. 1730bd9289cSQu Wenruo */ 1740bd9289cSQu Wenruo pending = atomic_read(&wq->pending); 1750bd9289cSQu Wenruo if (pending > wq->thresh) 1760bd9289cSQu Wenruo new_max_active++; 1770bd9289cSQu Wenruo if (pending < wq->thresh / 2) 1780bd9289cSQu Wenruo new_max_active--; 1790bd9289cSQu Wenruo new_max_active = clamp_val(new_max_active, 1, wq->max_active); 1800bd9289cSQu Wenruo if (new_max_active != wq->current_max) { 1810bd9289cSQu Wenruo need_change = 1; 1820bd9289cSQu Wenruo wq->current_max = new_max_active; 1830bd9289cSQu Wenruo } 1840bd9289cSQu Wenruo out: 1850bd9289cSQu Wenruo spin_unlock(&wq->thres_lock); 1860bd9289cSQu Wenruo 1870bd9289cSQu Wenruo if (need_change) { 1880bd9289cSQu Wenruo workqueue_set_max_active(wq->normal_wq, wq->current_max); 1890bd9289cSQu Wenruo } 1900bd9289cSQu Wenruo } 1910bd9289cSQu Wenruo 192d458b054SQu Wenruo static void run_ordered_work(struct __btrfs_workqueue *wq) 19308a9ff32SQu Wenruo { 19408a9ff32SQu Wenruo struct list_head *list = &wq->ordered_list; 195d458b054SQu Wenruo struct btrfs_work *work; 19608a9ff32SQu Wenruo spinlock_t *lock = &wq->list_lock; 19708a9ff32SQu Wenruo unsigned long flags; 19808a9ff32SQu Wenruo 19908a9ff32SQu Wenruo while (1) { 20008a9ff32SQu Wenruo spin_lock_irqsave(lock, flags); 20108a9ff32SQu Wenruo if (list_empty(list)) 20208a9ff32SQu Wenruo break; 203d458b054SQu Wenruo work = list_entry(list->next, struct btrfs_work, 20408a9ff32SQu Wenruo ordered_list); 20508a9ff32SQu Wenruo if (!test_bit(WORK_DONE_BIT, &work->flags)) 20608a9ff32SQu Wenruo break; 20708a9ff32SQu Wenruo 20808a9ff32SQu Wenruo /* 20908a9ff32SQu Wenruo * we are going to call the ordered done function, but 21008a9ff32SQu Wenruo * we leave the work item on the list as a barrier so 21108a9ff32SQu Wenruo * that later work items that are done don't have their 21208a9ff32SQu Wenruo * functions called before this one returns 21308a9ff32SQu Wenruo */ 21408a9ff32SQu Wenruo if (test_and_set_bit(WORK_ORDER_DONE_BIT, &work->flags)) 21508a9ff32SQu Wenruo break; 21652483bc2SQu Wenruo trace_btrfs_ordered_sched(work); 21708a9ff32SQu Wenruo spin_unlock_irqrestore(lock, flags); 21808a9ff32SQu Wenruo work->ordered_func(work); 21908a9ff32SQu Wenruo 22008a9ff32SQu Wenruo /* now take the lock again and drop our item from the list */ 22108a9ff32SQu Wenruo spin_lock_irqsave(lock, flags); 22208a9ff32SQu Wenruo list_del(&work->ordered_list); 22308a9ff32SQu Wenruo spin_unlock_irqrestore(lock, flags); 22408a9ff32SQu Wenruo 22508a9ff32SQu Wenruo /* 22608a9ff32SQu Wenruo * we don't want to call the ordered free functions 22708a9ff32SQu Wenruo * with the lock held though 22808a9ff32SQu Wenruo */ 22908a9ff32SQu Wenruo work->ordered_free(work); 23052483bc2SQu Wenruo trace_btrfs_all_work_done(work); 23108a9ff32SQu Wenruo } 23208a9ff32SQu Wenruo spin_unlock_irqrestore(lock, flags); 23308a9ff32SQu Wenruo } 23408a9ff32SQu Wenruo 23508a9ff32SQu Wenruo static void normal_work_helper(struct work_struct *arg) 23608a9ff32SQu Wenruo { 237d458b054SQu Wenruo struct btrfs_work *work; 238d458b054SQu Wenruo struct __btrfs_workqueue *wq; 23908a9ff32SQu Wenruo int need_order = 0; 24008a9ff32SQu Wenruo 241d458b054SQu Wenruo work = container_of(arg, struct btrfs_work, normal_work); 24208a9ff32SQu Wenruo /* 24308a9ff32SQu Wenruo * We should not touch things inside work in the following cases: 24408a9ff32SQu Wenruo * 1) after work->func() if it has no ordered_free 24508a9ff32SQu Wenruo * Since the struct is freed in work->func(). 24608a9ff32SQu Wenruo * 2) after setting WORK_DONE_BIT 24708a9ff32SQu Wenruo * The work may be freed in other threads almost instantly. 24808a9ff32SQu Wenruo * So we save the needed things here. 24908a9ff32SQu Wenruo */ 25008a9ff32SQu Wenruo if (work->ordered_func) 25108a9ff32SQu Wenruo need_order = 1; 25208a9ff32SQu Wenruo wq = work->wq; 25308a9ff32SQu Wenruo 25452483bc2SQu Wenruo trace_btrfs_work_sched(work); 2550bd9289cSQu Wenruo thresh_exec_hook(wq); 25608a9ff32SQu Wenruo work->func(work); 25708a9ff32SQu Wenruo if (need_order) { 25808a9ff32SQu Wenruo set_bit(WORK_DONE_BIT, &work->flags); 25908a9ff32SQu Wenruo run_ordered_work(wq); 26008a9ff32SQu Wenruo } 26152483bc2SQu Wenruo if (!need_order) 26252483bc2SQu Wenruo trace_btrfs_all_work_done(work); 26308a9ff32SQu Wenruo } 26408a9ff32SQu Wenruo 265d458b054SQu Wenruo void btrfs_init_work(struct btrfs_work *work, 2666db8914fSQu Wenruo btrfs_func_t func, 2676db8914fSQu Wenruo btrfs_func_t ordered_func, 2686db8914fSQu Wenruo btrfs_func_t ordered_free) 26908a9ff32SQu Wenruo { 27008a9ff32SQu Wenruo work->func = func; 27108a9ff32SQu Wenruo work->ordered_func = ordered_func; 27208a9ff32SQu Wenruo work->ordered_free = ordered_free; 27308a9ff32SQu Wenruo INIT_WORK(&work->normal_work, normal_work_helper); 27408a9ff32SQu Wenruo INIT_LIST_HEAD(&work->ordered_list); 27508a9ff32SQu Wenruo work->flags = 0; 27608a9ff32SQu Wenruo } 27708a9ff32SQu Wenruo 278d458b054SQu Wenruo static inline void __btrfs_queue_work(struct __btrfs_workqueue *wq, 279d458b054SQu Wenruo struct btrfs_work *work) 28008a9ff32SQu Wenruo { 28108a9ff32SQu Wenruo unsigned long flags; 28208a9ff32SQu Wenruo 28308a9ff32SQu Wenruo work->wq = wq; 2840bd9289cSQu Wenruo thresh_queue_hook(wq); 28508a9ff32SQu Wenruo if (work->ordered_func) { 28608a9ff32SQu Wenruo spin_lock_irqsave(&wq->list_lock, flags); 28708a9ff32SQu Wenruo list_add_tail(&work->ordered_list, &wq->ordered_list); 28808a9ff32SQu Wenruo spin_unlock_irqrestore(&wq->list_lock, flags); 28908a9ff32SQu Wenruo } 29008a9ff32SQu Wenruo queue_work(wq->normal_wq, &work->normal_work); 29152483bc2SQu Wenruo trace_btrfs_work_queued(work); 29208a9ff32SQu Wenruo } 29308a9ff32SQu Wenruo 294d458b054SQu Wenruo void btrfs_queue_work(struct btrfs_workqueue *wq, 295d458b054SQu Wenruo struct btrfs_work *work) 2961ca08976SQu Wenruo { 297d458b054SQu Wenruo struct __btrfs_workqueue *dest_wq; 2981ca08976SQu Wenruo 2991ca08976SQu Wenruo if (test_bit(WORK_HIGH_PRIO_BIT, &work->flags) && wq->high) 3001ca08976SQu Wenruo dest_wq = wq->high; 3011ca08976SQu Wenruo else 3021ca08976SQu Wenruo dest_wq = wq->normal; 3031ca08976SQu Wenruo __btrfs_queue_work(dest_wq, work); 3041ca08976SQu Wenruo } 3051ca08976SQu Wenruo 3061ca08976SQu Wenruo static inline void 307d458b054SQu Wenruo __btrfs_destroy_workqueue(struct __btrfs_workqueue *wq) 30808a9ff32SQu Wenruo { 30908a9ff32SQu Wenruo destroy_workqueue(wq->normal_wq); 310c3a46891SQu Wenruo trace_btrfs_workqueue_destroy(wq); 31108a9ff32SQu Wenruo kfree(wq); 31208a9ff32SQu Wenruo } 31308a9ff32SQu Wenruo 314d458b054SQu Wenruo void btrfs_destroy_workqueue(struct btrfs_workqueue *wq) 3151ca08976SQu Wenruo { 3161ca08976SQu Wenruo if (!wq) 3171ca08976SQu Wenruo return; 3181ca08976SQu Wenruo if (wq->high) 3191ca08976SQu Wenruo __btrfs_destroy_workqueue(wq->high); 3201ca08976SQu Wenruo __btrfs_destroy_workqueue(wq->normal); 321ef66af10SFilipe Manana kfree(wq); 3221ca08976SQu Wenruo } 3231ca08976SQu Wenruo 324d458b054SQu Wenruo void btrfs_workqueue_set_max(struct btrfs_workqueue *wq, int max) 32508a9ff32SQu Wenruo { 3260bd9289cSQu Wenruo wq->normal->max_active = max; 3271ca08976SQu Wenruo if (wq->high) 3280bd9289cSQu Wenruo wq->high->max_active = max; 3291ca08976SQu Wenruo } 3301ca08976SQu Wenruo 331d458b054SQu Wenruo void btrfs_set_work_high_priority(struct btrfs_work *work) 3321ca08976SQu Wenruo { 3331ca08976SQu Wenruo set_bit(WORK_HIGH_PRIO_BIT, &work->flags); 33408a9ff32SQu Wenruo } 335