11da177e4SLinus Torvalds /* 21da177e4SLinus Torvalds * Copyright (C) 2003 Sistina Software Limited. 31da177e4SLinus Torvalds * Copyright (C) 2004-2005 Red Hat, Inc. All rights reserved. 41da177e4SLinus Torvalds * 51da177e4SLinus Torvalds * This file is released under the GPL. 61da177e4SLinus Torvalds */ 71da177e4SLinus Torvalds 8586e80e6SMikulas Patocka #include <linux/device-mapper.h> 9586e80e6SMikulas Patocka 10f4790826SMike Snitzer #include "dm.h" 111da177e4SLinus Torvalds #include "dm-path-selector.h" 12b15546f9SMike Anderson #include "dm-uevent.h" 131da177e4SLinus Torvalds 141da177e4SLinus Torvalds #include <linux/ctype.h> 151da177e4SLinus Torvalds #include <linux/init.h> 161da177e4SLinus Torvalds #include <linux/mempool.h> 171da177e4SLinus Torvalds #include <linux/module.h> 181da177e4SLinus Torvalds #include <linux/pagemap.h> 191da177e4SLinus Torvalds #include <linux/slab.h> 201da177e4SLinus Torvalds #include <linux/time.h> 211da177e4SLinus Torvalds #include <linux/workqueue.h> 2235991652SMikulas Patocka #include <linux/delay.h> 23cfae5c9bSChandra Seetharaman #include <scsi/scsi_dh.h> 2460063497SArun Sharma #include <linux/atomic.h> 251da177e4SLinus Torvalds 2672d94861SAlasdair G Kergon #define DM_MSG_PREFIX "multipath" 274e2d19e4SChandra Seetharaman #define DM_PG_INIT_DELAY_MSECS 2000 284e2d19e4SChandra Seetharaman #define DM_PG_INIT_DELAY_DEFAULT ((unsigned) -1) 291da177e4SLinus Torvalds 301da177e4SLinus Torvalds /* Path properties */ 311da177e4SLinus Torvalds struct pgpath { 321da177e4SLinus Torvalds struct list_head list; 331da177e4SLinus Torvalds 341da177e4SLinus Torvalds struct priority_group *pg; /* Owning PG */ 356680073dSKiyoshi Ueda unsigned is_active; /* Path status */ 361da177e4SLinus Torvalds unsigned fail_count; /* Cumulative failure count */ 371da177e4SLinus Torvalds 38c922d5f7SJosef "Jeff" Sipek struct dm_path path; 394e2d19e4SChandra Seetharaman struct delayed_work activate_path; 401da177e4SLinus Torvalds }; 411da177e4SLinus Torvalds 421da177e4SLinus Torvalds #define path_to_pgpath(__pgp) container_of((__pgp), struct pgpath, path) 431da177e4SLinus Torvalds 441da177e4SLinus Torvalds /* 451da177e4SLinus Torvalds * Paths are grouped into Priority Groups and numbered from 1 upwards. 461da177e4SLinus Torvalds * Each has a path selector which controls which path gets used. 471da177e4SLinus Torvalds */ 481da177e4SLinus Torvalds struct priority_group { 491da177e4SLinus Torvalds struct list_head list; 501da177e4SLinus Torvalds 511da177e4SLinus Torvalds struct multipath *m; /* Owning multipath instance */ 521da177e4SLinus Torvalds struct path_selector ps; 531da177e4SLinus Torvalds 541da177e4SLinus Torvalds unsigned pg_num; /* Reference number */ 551da177e4SLinus Torvalds unsigned bypassed; /* Temporarily bypass this PG? */ 561da177e4SLinus Torvalds 571da177e4SLinus Torvalds unsigned nr_pgpaths; /* Number of paths in PG */ 581da177e4SLinus Torvalds struct list_head pgpaths; 591da177e4SLinus Torvalds }; 601da177e4SLinus Torvalds 611da177e4SLinus Torvalds /* Multipath context */ 621da177e4SLinus Torvalds struct multipath { 631da177e4SLinus Torvalds struct list_head list; 641da177e4SLinus Torvalds struct dm_target *ti; 651da177e4SLinus Torvalds 66cfae5c9bSChandra Seetharaman const char *hw_handler_name; 672bfd2e13SChandra Seetharaman char *hw_handler_params; 684e2d19e4SChandra Seetharaman 691fbdd2b3SMike Snitzer spinlock_t lock; 701fbdd2b3SMike Snitzer 711da177e4SLinus Torvalds unsigned nr_priority_groups; 721da177e4SLinus Torvalds struct list_head priority_groups; 734e2d19e4SChandra Seetharaman 744e2d19e4SChandra Seetharaman wait_queue_head_t pg_init_wait; /* Wait for pg_init completion */ 754e2d19e4SChandra Seetharaman 761da177e4SLinus Torvalds unsigned pg_init_required; /* pg_init needs calling? */ 77c3cd4f6bSAlasdair G Kergon unsigned pg_init_in_progress; /* Only one pg_init allowed at once */ 784e2d19e4SChandra Seetharaman unsigned pg_init_delay_retry; /* Delay pg_init retry? */ 791da177e4SLinus Torvalds 801da177e4SLinus Torvalds unsigned nr_valid_paths; /* Total number of usable paths */ 811da177e4SLinus Torvalds struct pgpath *current_pgpath; 821da177e4SLinus Torvalds struct priority_group *current_pg; 831da177e4SLinus Torvalds struct priority_group *next_pg; /* Switch to this PG if set */ 841da177e4SLinus Torvalds unsigned repeat_count; /* I/Os left before calling PS again */ 851da177e4SLinus Torvalds 861fbdd2b3SMike Snitzer unsigned queue_io:1; /* Must we queue all I/O? */ 871fbdd2b3SMike Snitzer unsigned queue_if_no_path:1; /* Queue I/O if last path fails? */ 881fbdd2b3SMike Snitzer unsigned saved_queue_if_no_path:1; /* Saved state during suspension */ 89a58a935dSMike Snitzer unsigned retain_attached_hw_handler:1; /* If there's already a hw_handler present, don't change it. */ 90954a73d5SShiva Krishna Merla unsigned pg_init_disabled:1; /* pg_init is not currently allowed */ 911fbdd2b3SMike Snitzer 92c9e45581SDave Wysochanski unsigned pg_init_retries; /* Number of times to retry pg_init */ 93c9e45581SDave Wysochanski unsigned pg_init_count; /* Number of times pg_init called */ 944e2d19e4SChandra Seetharaman unsigned pg_init_delay_msecs; /* Number of msecs before pg_init retry */ 951da177e4SLinus Torvalds 961da177e4SLinus Torvalds struct work_struct trigger_event; 971da177e4SLinus Torvalds 981da177e4SLinus Torvalds /* 99028867acSAlasdair G Kergon * We must use a mempool of dm_mpath_io structs so that we 1001da177e4SLinus Torvalds * can resubmit bios on error. 1011da177e4SLinus Torvalds */ 1021da177e4SLinus Torvalds mempool_t *mpio_pool; 1036380f26fSMike Anderson 1046380f26fSMike Anderson struct mutex work_mutex; 1051da177e4SLinus Torvalds }; 1061da177e4SLinus Torvalds 1071da177e4SLinus Torvalds /* 1081da177e4SLinus Torvalds * Context information attached to each bio we process. 1091da177e4SLinus Torvalds */ 110028867acSAlasdair G Kergon struct dm_mpath_io { 1111da177e4SLinus Torvalds struct pgpath *pgpath; 11202ab823fSKiyoshi Ueda size_t nr_bytes; 1131da177e4SLinus Torvalds }; 1141da177e4SLinus Torvalds 1151da177e4SLinus Torvalds typedef int (*action_fn) (struct pgpath *pgpath); 1161da177e4SLinus Torvalds 117e18b890bSChristoph Lameter static struct kmem_cache *_mpio_cache; 1181da177e4SLinus Torvalds 119bab7cfc7SChandra Seetharaman static struct workqueue_struct *kmultipathd, *kmpath_handlerd; 120c4028958SDavid Howells static void trigger_event(struct work_struct *work); 121bab7cfc7SChandra Seetharaman static void activate_path(struct work_struct *work); 122e8099177SHannes Reinecke static int __pgpath_busy(struct pgpath *pgpath); 1231da177e4SLinus Torvalds 1241da177e4SLinus Torvalds 1251da177e4SLinus Torvalds /*----------------------------------------------- 1261da177e4SLinus Torvalds * Allocation routines 1271da177e4SLinus Torvalds *-----------------------------------------------*/ 1281da177e4SLinus Torvalds 1291da177e4SLinus Torvalds static struct pgpath *alloc_pgpath(void) 1301da177e4SLinus Torvalds { 131e69fae56SMicha³ Miros³aw struct pgpath *pgpath = kzalloc(sizeof(*pgpath), GFP_KERNEL); 1321da177e4SLinus Torvalds 133224cb3e9SMike Anderson if (pgpath) { 1346680073dSKiyoshi Ueda pgpath->is_active = 1; 1354e2d19e4SChandra Seetharaman INIT_DELAYED_WORK(&pgpath->activate_path, activate_path); 136224cb3e9SMike Anderson } 1371da177e4SLinus Torvalds 1381da177e4SLinus Torvalds return pgpath; 1391da177e4SLinus Torvalds } 1401da177e4SLinus Torvalds 141028867acSAlasdair G Kergon static void free_pgpath(struct pgpath *pgpath) 1421da177e4SLinus Torvalds { 1431da177e4SLinus Torvalds kfree(pgpath); 1441da177e4SLinus Torvalds } 1451da177e4SLinus Torvalds 1461da177e4SLinus Torvalds static struct priority_group *alloc_priority_group(void) 1471da177e4SLinus Torvalds { 1481da177e4SLinus Torvalds struct priority_group *pg; 1491da177e4SLinus Torvalds 150e69fae56SMicha³ Miros³aw pg = kzalloc(sizeof(*pg), GFP_KERNEL); 1511da177e4SLinus Torvalds 152e69fae56SMicha³ Miros³aw if (pg) 1531da177e4SLinus Torvalds INIT_LIST_HEAD(&pg->pgpaths); 1541da177e4SLinus Torvalds 1551da177e4SLinus Torvalds return pg; 1561da177e4SLinus Torvalds } 1571da177e4SLinus Torvalds 1581da177e4SLinus Torvalds static void free_pgpaths(struct list_head *pgpaths, struct dm_target *ti) 1591da177e4SLinus Torvalds { 1601da177e4SLinus Torvalds struct pgpath *pgpath, *tmp; 161ae11b1b3SHannes Reinecke struct multipath *m = ti->private; 1621da177e4SLinus Torvalds 1631da177e4SLinus Torvalds list_for_each_entry_safe(pgpath, tmp, pgpaths, list) { 1641da177e4SLinus Torvalds list_del(&pgpath->list); 165ae11b1b3SHannes Reinecke if (m->hw_handler_name) 166ae11b1b3SHannes Reinecke scsi_dh_detach(bdev_get_queue(pgpath->path.dev->bdev)); 1671da177e4SLinus Torvalds dm_put_device(ti, pgpath->path.dev); 1681da177e4SLinus Torvalds free_pgpath(pgpath); 1691da177e4SLinus Torvalds } 1701da177e4SLinus Torvalds } 1711da177e4SLinus Torvalds 1721da177e4SLinus Torvalds static void free_priority_group(struct priority_group *pg, 1731da177e4SLinus Torvalds struct dm_target *ti) 1741da177e4SLinus Torvalds { 1751da177e4SLinus Torvalds struct path_selector *ps = &pg->ps; 1761da177e4SLinus Torvalds 1771da177e4SLinus Torvalds if (ps->type) { 1781da177e4SLinus Torvalds ps->type->destroy(ps); 1791da177e4SLinus Torvalds dm_put_path_selector(ps->type); 1801da177e4SLinus Torvalds } 1811da177e4SLinus Torvalds 1821da177e4SLinus Torvalds free_pgpaths(&pg->pgpaths, ti); 1831da177e4SLinus Torvalds kfree(pg); 1841da177e4SLinus Torvalds } 1851da177e4SLinus Torvalds 18628f16c20SMicha³ Miros³aw static struct multipath *alloc_multipath(struct dm_target *ti) 1871da177e4SLinus Torvalds { 1881da177e4SLinus Torvalds struct multipath *m; 189f4790826SMike Snitzer unsigned min_ios = dm_get_reserved_rq_based_ios(); 1901da177e4SLinus Torvalds 191e69fae56SMicha³ Miros³aw m = kzalloc(sizeof(*m), GFP_KERNEL); 1921da177e4SLinus Torvalds if (m) { 1931da177e4SLinus Torvalds INIT_LIST_HEAD(&m->priority_groups); 1941da177e4SLinus Torvalds spin_lock_init(&m->lock); 1951da177e4SLinus Torvalds m->queue_io = 1; 1964e2d19e4SChandra Seetharaman m->pg_init_delay_msecs = DM_PG_INIT_DELAY_DEFAULT; 197c4028958SDavid Howells INIT_WORK(&m->trigger_event, trigger_event); 1982bded7bdSKiyoshi Ueda init_waitqueue_head(&m->pg_init_wait); 1996380f26fSMike Anderson mutex_init(&m->work_mutex); 200f4790826SMike Snitzer m->mpio_pool = mempool_create_slab_pool(min_ios, _mpio_cache); 2011da177e4SLinus Torvalds if (!m->mpio_pool) { 2021da177e4SLinus Torvalds kfree(m); 2031da177e4SLinus Torvalds return NULL; 2041da177e4SLinus Torvalds } 20528f16c20SMicha³ Miros³aw m->ti = ti; 20628f16c20SMicha³ Miros³aw ti->private = m; 2071da177e4SLinus Torvalds } 2081da177e4SLinus Torvalds 2091da177e4SLinus Torvalds return m; 2101da177e4SLinus Torvalds } 2111da177e4SLinus Torvalds 2121da177e4SLinus Torvalds static void free_multipath(struct multipath *m) 2131da177e4SLinus Torvalds { 2141da177e4SLinus Torvalds struct priority_group *pg, *tmp; 2151da177e4SLinus Torvalds 2161da177e4SLinus Torvalds list_for_each_entry_safe(pg, tmp, &m->priority_groups, list) { 2171da177e4SLinus Torvalds list_del(&pg->list); 2181da177e4SLinus Torvalds free_priority_group(pg, m->ti); 2191da177e4SLinus Torvalds } 2201da177e4SLinus Torvalds 221cfae5c9bSChandra Seetharaman kfree(m->hw_handler_name); 2222bfd2e13SChandra Seetharaman kfree(m->hw_handler_params); 2231da177e4SLinus Torvalds mempool_destroy(m->mpio_pool); 2241da177e4SLinus Torvalds kfree(m); 2251da177e4SLinus Torvalds } 2261da177e4SLinus Torvalds 227466891f9SJun'ichi Nomura static int set_mapinfo(struct multipath *m, union map_info *info) 228466891f9SJun'ichi Nomura { 229466891f9SJun'ichi Nomura struct dm_mpath_io *mpio; 230466891f9SJun'ichi Nomura 231466891f9SJun'ichi Nomura mpio = mempool_alloc(m->mpio_pool, GFP_ATOMIC); 232466891f9SJun'ichi Nomura if (!mpio) 233466891f9SJun'ichi Nomura return -ENOMEM; 234466891f9SJun'ichi Nomura 235466891f9SJun'ichi Nomura memset(mpio, 0, sizeof(*mpio)); 236466891f9SJun'ichi Nomura info->ptr = mpio; 237466891f9SJun'ichi Nomura 238466891f9SJun'ichi Nomura return 0; 239466891f9SJun'ichi Nomura } 240466891f9SJun'ichi Nomura 241466891f9SJun'ichi Nomura static void clear_mapinfo(struct multipath *m, union map_info *info) 242466891f9SJun'ichi Nomura { 243466891f9SJun'ichi Nomura struct dm_mpath_io *mpio = info->ptr; 244466891f9SJun'ichi Nomura 245466891f9SJun'ichi Nomura info->ptr = NULL; 246466891f9SJun'ichi Nomura mempool_free(mpio, m->mpio_pool); 247466891f9SJun'ichi Nomura } 2481da177e4SLinus Torvalds 2491da177e4SLinus Torvalds /*----------------------------------------------- 2501da177e4SLinus Torvalds * Path selection 2511da177e4SLinus Torvalds *-----------------------------------------------*/ 2521da177e4SLinus Torvalds 2533e9f1be1SHannes Reinecke static int __pg_init_all_paths(struct multipath *m) 254fb612642SKiyoshi Ueda { 255fb612642SKiyoshi Ueda struct pgpath *pgpath; 2564e2d19e4SChandra Seetharaman unsigned long pg_init_delay = 0; 257fb612642SKiyoshi Ueda 25817f4ff45SHannes Reinecke if (m->pg_init_in_progress || m->pg_init_disabled) 2593e9f1be1SHannes Reinecke return 0; 26017f4ff45SHannes Reinecke 261fb612642SKiyoshi Ueda m->pg_init_count++; 262fb612642SKiyoshi Ueda m->pg_init_required = 0; 2633e9f1be1SHannes Reinecke 2643e9f1be1SHannes Reinecke /* Check here to reset pg_init_required */ 2653e9f1be1SHannes Reinecke if (!m->current_pg) 2663e9f1be1SHannes Reinecke return 0; 2673e9f1be1SHannes Reinecke 2684e2d19e4SChandra Seetharaman if (m->pg_init_delay_retry) 2694e2d19e4SChandra Seetharaman pg_init_delay = msecs_to_jiffies(m->pg_init_delay_msecs != DM_PG_INIT_DELAY_DEFAULT ? 2704e2d19e4SChandra Seetharaman m->pg_init_delay_msecs : DM_PG_INIT_DELAY_MSECS); 271fb612642SKiyoshi Ueda list_for_each_entry(pgpath, &m->current_pg->pgpaths, list) { 272fb612642SKiyoshi Ueda /* Skip failed paths */ 273fb612642SKiyoshi Ueda if (!pgpath->is_active) 274fb612642SKiyoshi Ueda continue; 2754e2d19e4SChandra Seetharaman if (queue_delayed_work(kmpath_handlerd, &pgpath->activate_path, 2764e2d19e4SChandra Seetharaman pg_init_delay)) 277fb612642SKiyoshi Ueda m->pg_init_in_progress++; 278fb612642SKiyoshi Ueda } 2793e9f1be1SHannes Reinecke return m->pg_init_in_progress; 280fb612642SKiyoshi Ueda } 281fb612642SKiyoshi Ueda 2821da177e4SLinus Torvalds static void __switch_pg(struct multipath *m, struct pgpath *pgpath) 2831da177e4SLinus Torvalds { 2841da177e4SLinus Torvalds m->current_pg = pgpath->pg; 2851da177e4SLinus Torvalds 2861da177e4SLinus Torvalds /* Must we initialise the PG first, and queue I/O till it's ready? */ 287cfae5c9bSChandra Seetharaman if (m->hw_handler_name) { 2881da177e4SLinus Torvalds m->pg_init_required = 1; 2891da177e4SLinus Torvalds m->queue_io = 1; 2901da177e4SLinus Torvalds } else { 2911da177e4SLinus Torvalds m->pg_init_required = 0; 2921da177e4SLinus Torvalds m->queue_io = 0; 2931da177e4SLinus Torvalds } 294c9e45581SDave Wysochanski 295c9e45581SDave Wysochanski m->pg_init_count = 0; 2961da177e4SLinus Torvalds } 2971da177e4SLinus Torvalds 29802ab823fSKiyoshi Ueda static int __choose_path_in_pg(struct multipath *m, struct priority_group *pg, 29902ab823fSKiyoshi Ueda size_t nr_bytes) 3001da177e4SLinus Torvalds { 301c922d5f7SJosef "Jeff" Sipek struct dm_path *path; 3021da177e4SLinus Torvalds 30302ab823fSKiyoshi Ueda path = pg->ps.type->select_path(&pg->ps, &m->repeat_count, nr_bytes); 3041da177e4SLinus Torvalds if (!path) 3051da177e4SLinus Torvalds return -ENXIO; 3061da177e4SLinus Torvalds 3071da177e4SLinus Torvalds m->current_pgpath = path_to_pgpath(path); 3081da177e4SLinus Torvalds 3091da177e4SLinus Torvalds if (m->current_pg != pg) 3101da177e4SLinus Torvalds __switch_pg(m, m->current_pgpath); 3111da177e4SLinus Torvalds 3121da177e4SLinus Torvalds return 0; 3131da177e4SLinus Torvalds } 3141da177e4SLinus Torvalds 31502ab823fSKiyoshi Ueda static void __choose_pgpath(struct multipath *m, size_t nr_bytes) 3161da177e4SLinus Torvalds { 3171da177e4SLinus Torvalds struct priority_group *pg; 3181da177e4SLinus Torvalds unsigned bypassed = 1; 3191da177e4SLinus Torvalds 3201da177e4SLinus Torvalds if (!m->nr_valid_paths) 3211da177e4SLinus Torvalds goto failed; 3221da177e4SLinus Torvalds 3231da177e4SLinus Torvalds /* Were we instructed to switch PG? */ 3241da177e4SLinus Torvalds if (m->next_pg) { 3251da177e4SLinus Torvalds pg = m->next_pg; 3261da177e4SLinus Torvalds m->next_pg = NULL; 32702ab823fSKiyoshi Ueda if (!__choose_path_in_pg(m, pg, nr_bytes)) 3281da177e4SLinus Torvalds return; 3291da177e4SLinus Torvalds } 3301da177e4SLinus Torvalds 3311da177e4SLinus Torvalds /* Don't change PG until it has no remaining paths */ 33202ab823fSKiyoshi Ueda if (m->current_pg && !__choose_path_in_pg(m, m->current_pg, nr_bytes)) 3331da177e4SLinus Torvalds return; 3341da177e4SLinus Torvalds 3351da177e4SLinus Torvalds /* 3361da177e4SLinus Torvalds * Loop through priority groups until we find a valid path. 3371da177e4SLinus Torvalds * First time we skip PGs marked 'bypassed'. 338f220fd4eSMike Christie * Second time we only try the ones we skipped, but set 339f220fd4eSMike Christie * pg_init_delay_retry so we do not hammer controllers. 3401da177e4SLinus Torvalds */ 3411da177e4SLinus Torvalds do { 3421da177e4SLinus Torvalds list_for_each_entry(pg, &m->priority_groups, list) { 3431da177e4SLinus Torvalds if (pg->bypassed == bypassed) 3441da177e4SLinus Torvalds continue; 345f220fd4eSMike Christie if (!__choose_path_in_pg(m, pg, nr_bytes)) { 346f220fd4eSMike Christie if (!bypassed) 347f220fd4eSMike Christie m->pg_init_delay_retry = 1; 3481da177e4SLinus Torvalds return; 3491da177e4SLinus Torvalds } 350f220fd4eSMike Christie } 3511da177e4SLinus Torvalds } while (bypassed--); 3521da177e4SLinus Torvalds 3531da177e4SLinus Torvalds failed: 3541da177e4SLinus Torvalds m->current_pgpath = NULL; 3551da177e4SLinus Torvalds m->current_pg = NULL; 3561da177e4SLinus Torvalds } 3571da177e4SLinus Torvalds 35845e15720SKiyoshi Ueda /* 35945e15720SKiyoshi Ueda * Check whether bios must be queued in the device-mapper core rather 36045e15720SKiyoshi Ueda * than here in the target. 36145e15720SKiyoshi Ueda * 36245e15720SKiyoshi Ueda * m->lock must be held on entry. 36345e15720SKiyoshi Ueda * 36445e15720SKiyoshi Ueda * If m->queue_if_no_path and m->saved_queue_if_no_path hold the 36545e15720SKiyoshi Ueda * same value then we are not between multipath_presuspend() 36645e15720SKiyoshi Ueda * and multipath_resume() calls and we have no need to check 36745e15720SKiyoshi Ueda * for the DMF_NOFLUSH_SUSPENDING flag. 36845e15720SKiyoshi Ueda */ 36945e15720SKiyoshi Ueda static int __must_push_back(struct multipath *m) 37045e15720SKiyoshi Ueda { 371e8099177SHannes Reinecke return (m->queue_if_no_path || 372e8099177SHannes Reinecke (m->queue_if_no_path != m->saved_queue_if_no_path && 373e8099177SHannes Reinecke dm_noflush_suspending(m->ti))); 37445e15720SKiyoshi Ueda } 37545e15720SKiyoshi Ueda 376e8099177SHannes Reinecke #define pg_ready(m) (!(m)->queue_io && !(m)->pg_init_required) 377e8099177SHannes Reinecke 37836fcffccSHannes Reinecke /* 37936fcffccSHannes Reinecke * Map cloned requests 38036fcffccSHannes Reinecke */ 38136fcffccSHannes Reinecke static int multipath_map(struct dm_target *ti, struct request *clone, 382e8099177SHannes Reinecke union map_info *map_context) 3831da177e4SLinus Torvalds { 38436fcffccSHannes Reinecke struct multipath *m = (struct multipath *) ti->private; 385e3bde04fSHannes Reinecke int r = DM_MAPIO_REQUEUE; 386f40c67f0SKiyoshi Ueda size_t nr_bytes = blk_rq_bytes(clone); 3871da177e4SLinus Torvalds unsigned long flags; 3881da177e4SLinus Torvalds struct pgpath *pgpath; 389f40c67f0SKiyoshi Ueda struct block_device *bdev; 390e3bde04fSHannes Reinecke struct dm_mpath_io *mpio; 3911da177e4SLinus Torvalds 3921da177e4SLinus Torvalds spin_lock_irqsave(&m->lock, flags); 3931da177e4SLinus Torvalds 3941da177e4SLinus Torvalds /* Do we need to select a new pgpath? */ 3951da177e4SLinus Torvalds if (!m->current_pgpath || 3961da177e4SLinus Torvalds (!m->queue_io && (m->repeat_count && --m->repeat_count == 0))) 39702ab823fSKiyoshi Ueda __choose_pgpath(m, nr_bytes); 3981da177e4SLinus Torvalds 3991da177e4SLinus Torvalds pgpath = m->current_pgpath; 4001da177e4SLinus Torvalds 4019bf59a61SMike Snitzer if (!pgpath) { 4029bf59a61SMike Snitzer if (!__must_push_back(m)) 4039bf59a61SMike Snitzer r = -EIO; /* Failed */ 4049bf59a61SMike Snitzer goto out_unlock; 4059bf59a61SMike Snitzer } 4069bf59a61SMike Snitzer if (!pg_ready(m)) { 4079bf59a61SMike Snitzer __pg_init_all_paths(m); 4089bf59a61SMike Snitzer goto out_unlock; 4099bf59a61SMike Snitzer } 410e3bde04fSHannes Reinecke if (set_mapinfo(m, map_context) < 0) 411e3bde04fSHannes Reinecke /* ENOMEM, requeue */ 412e3bde04fSHannes Reinecke goto out_unlock; 413e3bde04fSHannes Reinecke 414f40c67f0SKiyoshi Ueda bdev = pgpath->path.dev->bdev; 415f40c67f0SKiyoshi Ueda clone->q = bdev_get_queue(bdev); 416f40c67f0SKiyoshi Ueda clone->rq_disk = bdev->bd_disk; 417e3bde04fSHannes Reinecke clone->cmd_flags |= REQ_FAILFAST_TRANSPORT; 418e3bde04fSHannes Reinecke mpio = map_context->ptr; 419e8099177SHannes Reinecke mpio->pgpath = pgpath; 420e8099177SHannes Reinecke mpio->nr_bytes = nr_bytes; 421e8099177SHannes Reinecke if (pgpath->pg->ps.type->start_io) 422e8099177SHannes Reinecke pgpath->pg->ps.type->start_io(&pgpath->pg->ps, 423e8099177SHannes Reinecke &pgpath->path, 424e8099177SHannes Reinecke nr_bytes); 425e3bde04fSHannes Reinecke r = DM_MAPIO_REMAPPED; 4261da177e4SLinus Torvalds 427e3bde04fSHannes Reinecke out_unlock: 4281da177e4SLinus Torvalds spin_unlock_irqrestore(&m->lock, flags); 4291da177e4SLinus Torvalds 4301da177e4SLinus Torvalds return r; 4311da177e4SLinus Torvalds } 4321da177e4SLinus Torvalds 4331da177e4SLinus Torvalds /* 4341da177e4SLinus Torvalds * If we run out of usable paths, should we queue I/O or error it? 4351da177e4SLinus Torvalds */ 436485ef69eSAlasdair G Kergon static int queue_if_no_path(struct multipath *m, unsigned queue_if_no_path, 437485ef69eSAlasdair G Kergon unsigned save_old_value) 4381da177e4SLinus Torvalds { 4391da177e4SLinus Torvalds unsigned long flags; 4401da177e4SLinus Torvalds 4411da177e4SLinus Torvalds spin_lock_irqsave(&m->lock, flags); 4421da177e4SLinus Torvalds 443485ef69eSAlasdair G Kergon if (save_old_value) 444436d4108SAlasdair G Kergon m->saved_queue_if_no_path = m->queue_if_no_path; 445485ef69eSAlasdair G Kergon else 446485ef69eSAlasdair G Kergon m->saved_queue_if_no_path = queue_if_no_path; 4471da177e4SLinus Torvalds m->queue_if_no_path = queue_if_no_path; 448e8099177SHannes Reinecke if (!m->queue_if_no_path) 4493e9f1be1SHannes Reinecke dm_table_run_md_queue_async(m->ti->table); 4501da177e4SLinus Torvalds 4511da177e4SLinus Torvalds spin_unlock_irqrestore(&m->lock, flags); 4521da177e4SLinus Torvalds 4531da177e4SLinus Torvalds return 0; 4541da177e4SLinus Torvalds } 4551da177e4SLinus Torvalds 4561da177e4SLinus Torvalds /* 4571da177e4SLinus Torvalds * An event is triggered whenever a path is taken out of use. 4581da177e4SLinus Torvalds * Includes path failure and PG bypass. 4591da177e4SLinus Torvalds */ 460c4028958SDavid Howells static void trigger_event(struct work_struct *work) 4611da177e4SLinus Torvalds { 462c4028958SDavid Howells struct multipath *m = 463c4028958SDavid Howells container_of(work, struct multipath, trigger_event); 4641da177e4SLinus Torvalds 4651da177e4SLinus Torvalds dm_table_event(m->ti->table); 4661da177e4SLinus Torvalds } 4671da177e4SLinus Torvalds 4681da177e4SLinus Torvalds /*----------------------------------------------------------------- 4691da177e4SLinus Torvalds * Constructor/argument parsing: 4701da177e4SLinus Torvalds * <#multipath feature args> [<arg>]* 4711da177e4SLinus Torvalds * <#hw_handler args> [hw_handler [<arg>]*] 4721da177e4SLinus Torvalds * <#priority groups> 4731da177e4SLinus Torvalds * <initial priority group> 4741da177e4SLinus Torvalds * [<selector> <#selector args> [<arg>]* 4751da177e4SLinus Torvalds * <#paths> <#per-path selector args> 4761da177e4SLinus Torvalds * [<path> [<arg>]* ]+ ]+ 4771da177e4SLinus Torvalds *---------------------------------------------------------------*/ 478498f0103SMike Snitzer static int parse_path_selector(struct dm_arg_set *as, struct priority_group *pg, 4791da177e4SLinus Torvalds struct dm_target *ti) 4801da177e4SLinus Torvalds { 4811da177e4SLinus Torvalds int r; 4821da177e4SLinus Torvalds struct path_selector_type *pst; 4831da177e4SLinus Torvalds unsigned ps_argc; 4841da177e4SLinus Torvalds 485498f0103SMike Snitzer static struct dm_arg _args[] = { 48672d94861SAlasdair G Kergon {0, 1024, "invalid number of path selector args"}, 4871da177e4SLinus Torvalds }; 4881da177e4SLinus Torvalds 489498f0103SMike Snitzer pst = dm_get_path_selector(dm_shift_arg(as)); 4901da177e4SLinus Torvalds if (!pst) { 49172d94861SAlasdair G Kergon ti->error = "unknown path selector type"; 4921da177e4SLinus Torvalds return -EINVAL; 4931da177e4SLinus Torvalds } 4941da177e4SLinus Torvalds 495498f0103SMike Snitzer r = dm_read_arg_group(_args, as, &ps_argc, &ti->error); 496371b2e34SMikulas Patocka if (r) { 497371b2e34SMikulas Patocka dm_put_path_selector(pst); 4981da177e4SLinus Torvalds return -EINVAL; 499371b2e34SMikulas Patocka } 5001da177e4SLinus Torvalds 5011da177e4SLinus Torvalds r = pst->create(&pg->ps, ps_argc, as->argv); 5021da177e4SLinus Torvalds if (r) { 5031da177e4SLinus Torvalds dm_put_path_selector(pst); 50472d94861SAlasdair G Kergon ti->error = "path selector constructor failed"; 5051da177e4SLinus Torvalds return r; 5061da177e4SLinus Torvalds } 5071da177e4SLinus Torvalds 5081da177e4SLinus Torvalds pg->ps.type = pst; 509498f0103SMike Snitzer dm_consume_args(as, ps_argc); 5101da177e4SLinus Torvalds 5111da177e4SLinus Torvalds return 0; 5121da177e4SLinus Torvalds } 5131da177e4SLinus Torvalds 514498f0103SMike Snitzer static struct pgpath *parse_path(struct dm_arg_set *as, struct path_selector *ps, 5151da177e4SLinus Torvalds struct dm_target *ti) 5161da177e4SLinus Torvalds { 5171da177e4SLinus Torvalds int r; 5181da177e4SLinus Torvalds struct pgpath *p; 519ae11b1b3SHannes Reinecke struct multipath *m = ti->private; 520a58a935dSMike Snitzer struct request_queue *q = NULL; 521a58a935dSMike Snitzer const char *attached_handler_name; 5221da177e4SLinus Torvalds 5231da177e4SLinus Torvalds /* we need at least a path arg */ 5241da177e4SLinus Torvalds if (as->argc < 1) { 52572d94861SAlasdair G Kergon ti->error = "no device given"; 52601460f35SBenjamin Marzinski return ERR_PTR(-EINVAL); 5271da177e4SLinus Torvalds } 5281da177e4SLinus Torvalds 5291da177e4SLinus Torvalds p = alloc_pgpath(); 5301da177e4SLinus Torvalds if (!p) 53101460f35SBenjamin Marzinski return ERR_PTR(-ENOMEM); 5321da177e4SLinus Torvalds 533498f0103SMike Snitzer r = dm_get_device(ti, dm_shift_arg(as), dm_table_get_mode(ti->table), 5348215d6ecSNikanth Karthikesan &p->path.dev); 5351da177e4SLinus Torvalds if (r) { 53672d94861SAlasdair G Kergon ti->error = "error getting device"; 5371da177e4SLinus Torvalds goto bad; 5381da177e4SLinus Torvalds } 5391da177e4SLinus Torvalds 540a58a935dSMike Snitzer if (m->retain_attached_hw_handler || m->hw_handler_name) 541a58a935dSMike Snitzer q = bdev_get_queue(p->path.dev->bdev); 542a0cf7ea9SHannes Reinecke 543a58a935dSMike Snitzer if (m->retain_attached_hw_handler) { 544a58a935dSMike Snitzer attached_handler_name = scsi_dh_attached_handler_name(q, GFP_KERNEL); 545a58a935dSMike Snitzer if (attached_handler_name) { 546a58a935dSMike Snitzer /* 547a58a935dSMike Snitzer * Reset hw_handler_name to match the attached handler 548a58a935dSMike Snitzer * and clear any hw_handler_params associated with the 549a58a935dSMike Snitzer * ignored handler. 550a58a935dSMike Snitzer * 551a58a935dSMike Snitzer * NB. This modifies the table line to show the actual 552a58a935dSMike Snitzer * handler instead of the original table passed in. 553a58a935dSMike Snitzer */ 554a58a935dSMike Snitzer kfree(m->hw_handler_name); 555a58a935dSMike Snitzer m->hw_handler_name = attached_handler_name; 556a58a935dSMike Snitzer 557a58a935dSMike Snitzer kfree(m->hw_handler_params); 558a58a935dSMike Snitzer m->hw_handler_params = NULL; 559a58a935dSMike Snitzer } 560a58a935dSMike Snitzer } 561a58a935dSMike Snitzer 562a58a935dSMike Snitzer if (m->hw_handler_name) { 563a58a935dSMike Snitzer /* 564a58a935dSMike Snitzer * Increments scsi_dh reference, even when using an 565a58a935dSMike Snitzer * already-attached handler. 566a58a935dSMike Snitzer */ 567a0cf7ea9SHannes Reinecke r = scsi_dh_attach(q, m->hw_handler_name); 568a0cf7ea9SHannes Reinecke if (r == -EBUSY) { 569a0cf7ea9SHannes Reinecke /* 570a58a935dSMike Snitzer * Already attached to different hw_handler: 571a0cf7ea9SHannes Reinecke * try to reattach with correct one. 572a0cf7ea9SHannes Reinecke */ 573a0cf7ea9SHannes Reinecke scsi_dh_detach(q); 574a0cf7ea9SHannes Reinecke r = scsi_dh_attach(q, m->hw_handler_name); 575a0cf7ea9SHannes Reinecke } 576a0cf7ea9SHannes Reinecke 577ae11b1b3SHannes Reinecke if (r < 0) { 578a0cf7ea9SHannes Reinecke ti->error = "error attaching hardware handler"; 579ae11b1b3SHannes Reinecke dm_put_device(ti, p->path.dev); 580ae11b1b3SHannes Reinecke goto bad; 581ae11b1b3SHannes Reinecke } 5822bfd2e13SChandra Seetharaman 5832bfd2e13SChandra Seetharaman if (m->hw_handler_params) { 5842bfd2e13SChandra Seetharaman r = scsi_dh_set_params(q, m->hw_handler_params); 5852bfd2e13SChandra Seetharaman if (r < 0) { 5862bfd2e13SChandra Seetharaman ti->error = "unable to set hardware " 5872bfd2e13SChandra Seetharaman "handler parameters"; 5882bfd2e13SChandra Seetharaman scsi_dh_detach(q); 5892bfd2e13SChandra Seetharaman dm_put_device(ti, p->path.dev); 5902bfd2e13SChandra Seetharaman goto bad; 5912bfd2e13SChandra Seetharaman } 5922bfd2e13SChandra Seetharaman } 593ae11b1b3SHannes Reinecke } 594ae11b1b3SHannes Reinecke 5951da177e4SLinus Torvalds r = ps->type->add_path(ps, &p->path, as->argc, as->argv, &ti->error); 5961da177e4SLinus Torvalds if (r) { 5971da177e4SLinus Torvalds dm_put_device(ti, p->path.dev); 5981da177e4SLinus Torvalds goto bad; 5991da177e4SLinus Torvalds } 6001da177e4SLinus Torvalds 6011da177e4SLinus Torvalds return p; 6021da177e4SLinus Torvalds 6031da177e4SLinus Torvalds bad: 6041da177e4SLinus Torvalds free_pgpath(p); 60501460f35SBenjamin Marzinski return ERR_PTR(r); 6061da177e4SLinus Torvalds } 6071da177e4SLinus Torvalds 608498f0103SMike Snitzer static struct priority_group *parse_priority_group(struct dm_arg_set *as, 60928f16c20SMicha³ Miros³aw struct multipath *m) 6101da177e4SLinus Torvalds { 611498f0103SMike Snitzer static struct dm_arg _args[] = { 61272d94861SAlasdair G Kergon {1, 1024, "invalid number of paths"}, 61372d94861SAlasdair G Kergon {0, 1024, "invalid number of selector args"} 6141da177e4SLinus Torvalds }; 6151da177e4SLinus Torvalds 6161da177e4SLinus Torvalds int r; 617498f0103SMike Snitzer unsigned i, nr_selector_args, nr_args; 6181da177e4SLinus Torvalds struct priority_group *pg; 61928f16c20SMicha³ Miros³aw struct dm_target *ti = m->ti; 6201da177e4SLinus Torvalds 6211da177e4SLinus Torvalds if (as->argc < 2) { 6221da177e4SLinus Torvalds as->argc = 0; 62301460f35SBenjamin Marzinski ti->error = "not enough priority group arguments"; 62401460f35SBenjamin Marzinski return ERR_PTR(-EINVAL); 6251da177e4SLinus Torvalds } 6261da177e4SLinus Torvalds 6271da177e4SLinus Torvalds pg = alloc_priority_group(); 6281da177e4SLinus Torvalds if (!pg) { 62972d94861SAlasdair G Kergon ti->error = "couldn't allocate priority group"; 63001460f35SBenjamin Marzinski return ERR_PTR(-ENOMEM); 6311da177e4SLinus Torvalds } 6321da177e4SLinus Torvalds pg->m = m; 6331da177e4SLinus Torvalds 6341da177e4SLinus Torvalds r = parse_path_selector(as, pg, ti); 6351da177e4SLinus Torvalds if (r) 6361da177e4SLinus Torvalds goto bad; 6371da177e4SLinus Torvalds 6381da177e4SLinus Torvalds /* 6391da177e4SLinus Torvalds * read the paths 6401da177e4SLinus Torvalds */ 641498f0103SMike Snitzer r = dm_read_arg(_args, as, &pg->nr_pgpaths, &ti->error); 6421da177e4SLinus Torvalds if (r) 6431da177e4SLinus Torvalds goto bad; 6441da177e4SLinus Torvalds 645498f0103SMike Snitzer r = dm_read_arg(_args + 1, as, &nr_selector_args, &ti->error); 6461da177e4SLinus Torvalds if (r) 6471da177e4SLinus Torvalds goto bad; 6481da177e4SLinus Torvalds 649498f0103SMike Snitzer nr_args = 1 + nr_selector_args; 6501da177e4SLinus Torvalds for (i = 0; i < pg->nr_pgpaths; i++) { 6511da177e4SLinus Torvalds struct pgpath *pgpath; 652498f0103SMike Snitzer struct dm_arg_set path_args; 6531da177e4SLinus Torvalds 654498f0103SMike Snitzer if (as->argc < nr_args) { 655148acff6SMikulas Patocka ti->error = "not enough path parameters"; 6566bbf79a1SAlasdair G Kergon r = -EINVAL; 6571da177e4SLinus Torvalds goto bad; 658148acff6SMikulas Patocka } 6591da177e4SLinus Torvalds 660498f0103SMike Snitzer path_args.argc = nr_args; 6611da177e4SLinus Torvalds path_args.argv = as->argv; 6621da177e4SLinus Torvalds 6631da177e4SLinus Torvalds pgpath = parse_path(&path_args, &pg->ps, ti); 66401460f35SBenjamin Marzinski if (IS_ERR(pgpath)) { 66501460f35SBenjamin Marzinski r = PTR_ERR(pgpath); 6661da177e4SLinus Torvalds goto bad; 66701460f35SBenjamin Marzinski } 6681da177e4SLinus Torvalds 6691da177e4SLinus Torvalds pgpath->pg = pg; 6701da177e4SLinus Torvalds list_add_tail(&pgpath->list, &pg->pgpaths); 671498f0103SMike Snitzer dm_consume_args(as, nr_args); 6721da177e4SLinus Torvalds } 6731da177e4SLinus Torvalds 6741da177e4SLinus Torvalds return pg; 6751da177e4SLinus Torvalds 6761da177e4SLinus Torvalds bad: 6771da177e4SLinus Torvalds free_priority_group(pg, ti); 67801460f35SBenjamin Marzinski return ERR_PTR(r); 6791da177e4SLinus Torvalds } 6801da177e4SLinus Torvalds 681498f0103SMike Snitzer static int parse_hw_handler(struct dm_arg_set *as, struct multipath *m) 6821da177e4SLinus Torvalds { 6831da177e4SLinus Torvalds unsigned hw_argc; 6842bfd2e13SChandra Seetharaman int ret; 68528f16c20SMicha³ Miros³aw struct dm_target *ti = m->ti; 6861da177e4SLinus Torvalds 687498f0103SMike Snitzer static struct dm_arg _args[] = { 68872d94861SAlasdair G Kergon {0, 1024, "invalid number of hardware handler args"}, 6891da177e4SLinus Torvalds }; 6901da177e4SLinus Torvalds 691498f0103SMike Snitzer if (dm_read_arg_group(_args, as, &hw_argc, &ti->error)) 6921da177e4SLinus Torvalds return -EINVAL; 6931da177e4SLinus Torvalds 6941da177e4SLinus Torvalds if (!hw_argc) 6951da177e4SLinus Torvalds return 0; 6961da177e4SLinus Torvalds 697498f0103SMike Snitzer m->hw_handler_name = kstrdup(dm_shift_arg(as), GFP_KERNEL); 698510193a2SMike Snitzer if (!try_then_request_module(scsi_dh_handler_exist(m->hw_handler_name), 699510193a2SMike Snitzer "scsi_dh_%s", m->hw_handler_name)) { 70072d94861SAlasdair G Kergon ti->error = "unknown hardware handler type"; 7012bfd2e13SChandra Seetharaman ret = -EINVAL; 7022bfd2e13SChandra Seetharaman goto fail; 7031da177e4SLinus Torvalds } 70414e98c5cSChandra Seetharaman 7052bfd2e13SChandra Seetharaman if (hw_argc > 1) { 7062bfd2e13SChandra Seetharaman char *p; 7072bfd2e13SChandra Seetharaman int i, j, len = 4; 7082bfd2e13SChandra Seetharaman 7092bfd2e13SChandra Seetharaman for (i = 0; i <= hw_argc - 2; i++) 7102bfd2e13SChandra Seetharaman len += strlen(as->argv[i]) + 1; 7112bfd2e13SChandra Seetharaman p = m->hw_handler_params = kzalloc(len, GFP_KERNEL); 7122bfd2e13SChandra Seetharaman if (!p) { 7132bfd2e13SChandra Seetharaman ti->error = "memory allocation failed"; 7142bfd2e13SChandra Seetharaman ret = -ENOMEM; 7152bfd2e13SChandra Seetharaman goto fail; 7162bfd2e13SChandra Seetharaman } 7172bfd2e13SChandra Seetharaman j = sprintf(p, "%d", hw_argc - 1); 7182bfd2e13SChandra Seetharaman for (i = 0, p+=j+1; i <= hw_argc - 2; i++, p+=j+1) 7192bfd2e13SChandra Seetharaman j = sprintf(p, "%s", as->argv[i]); 7202bfd2e13SChandra Seetharaman } 721498f0103SMike Snitzer dm_consume_args(as, hw_argc - 1); 7221da177e4SLinus Torvalds 7231da177e4SLinus Torvalds return 0; 7242bfd2e13SChandra Seetharaman fail: 7252bfd2e13SChandra Seetharaman kfree(m->hw_handler_name); 7262bfd2e13SChandra Seetharaman m->hw_handler_name = NULL; 7272bfd2e13SChandra Seetharaman return ret; 7281da177e4SLinus Torvalds } 7291da177e4SLinus Torvalds 730498f0103SMike Snitzer static int parse_features(struct dm_arg_set *as, struct multipath *m) 7311da177e4SLinus Torvalds { 7321da177e4SLinus Torvalds int r; 7331da177e4SLinus Torvalds unsigned argc; 73428f16c20SMicha³ Miros³aw struct dm_target *ti = m->ti; 735498f0103SMike Snitzer const char *arg_name; 7361da177e4SLinus Torvalds 737498f0103SMike Snitzer static struct dm_arg _args[] = { 738a58a935dSMike Snitzer {0, 6, "invalid number of feature args"}, 739c9e45581SDave Wysochanski {1, 50, "pg_init_retries must be between 1 and 50"}, 7404e2d19e4SChandra Seetharaman {0, 60000, "pg_init_delay_msecs must be between 0 and 60000"}, 7411da177e4SLinus Torvalds }; 7421da177e4SLinus Torvalds 743498f0103SMike Snitzer r = dm_read_arg_group(_args, as, &argc, &ti->error); 7441da177e4SLinus Torvalds if (r) 7451da177e4SLinus Torvalds return -EINVAL; 7461da177e4SLinus Torvalds 7471da177e4SLinus Torvalds if (!argc) 7481da177e4SLinus Torvalds return 0; 7491da177e4SLinus Torvalds 750c9e45581SDave Wysochanski do { 751498f0103SMike Snitzer arg_name = dm_shift_arg(as); 752c9e45581SDave Wysochanski argc--; 753c9e45581SDave Wysochanski 754498f0103SMike Snitzer if (!strcasecmp(arg_name, "queue_if_no_path")) { 755c9e45581SDave Wysochanski r = queue_if_no_path(m, 1, 0); 756c9e45581SDave Wysochanski continue; 7571da177e4SLinus Torvalds } 758c9e45581SDave Wysochanski 759a58a935dSMike Snitzer if (!strcasecmp(arg_name, "retain_attached_hw_handler")) { 760a58a935dSMike Snitzer m->retain_attached_hw_handler = 1; 761a58a935dSMike Snitzer continue; 762a58a935dSMike Snitzer } 763a58a935dSMike Snitzer 764498f0103SMike Snitzer if (!strcasecmp(arg_name, "pg_init_retries") && 765c9e45581SDave Wysochanski (argc >= 1)) { 766498f0103SMike Snitzer r = dm_read_arg(_args + 1, as, &m->pg_init_retries, &ti->error); 767c9e45581SDave Wysochanski argc--; 768c9e45581SDave Wysochanski continue; 769c9e45581SDave Wysochanski } 770c9e45581SDave Wysochanski 771498f0103SMike Snitzer if (!strcasecmp(arg_name, "pg_init_delay_msecs") && 7724e2d19e4SChandra Seetharaman (argc >= 1)) { 773498f0103SMike Snitzer r = dm_read_arg(_args + 2, as, &m->pg_init_delay_msecs, &ti->error); 7744e2d19e4SChandra Seetharaman argc--; 7754e2d19e4SChandra Seetharaman continue; 7764e2d19e4SChandra Seetharaman } 7774e2d19e4SChandra Seetharaman 778c9e45581SDave Wysochanski ti->error = "Unrecognised multipath feature request"; 779c9e45581SDave Wysochanski r = -EINVAL; 780c9e45581SDave Wysochanski } while (argc && !r); 781c9e45581SDave Wysochanski 782c9e45581SDave Wysochanski return r; 7831da177e4SLinus Torvalds } 7841da177e4SLinus Torvalds 7851da177e4SLinus Torvalds static int multipath_ctr(struct dm_target *ti, unsigned int argc, 7861da177e4SLinus Torvalds char **argv) 7871da177e4SLinus Torvalds { 788498f0103SMike Snitzer /* target arguments */ 789498f0103SMike Snitzer static struct dm_arg _args[] = { 790a490a07aSMike Snitzer {0, 1024, "invalid number of priority groups"}, 791a490a07aSMike Snitzer {0, 1024, "invalid initial priority group number"}, 7921da177e4SLinus Torvalds }; 7931da177e4SLinus Torvalds 7941da177e4SLinus Torvalds int r; 7951da177e4SLinus Torvalds struct multipath *m; 796498f0103SMike Snitzer struct dm_arg_set as; 7971da177e4SLinus Torvalds unsigned pg_count = 0; 7981da177e4SLinus Torvalds unsigned next_pg_num; 7991da177e4SLinus Torvalds 8001da177e4SLinus Torvalds as.argc = argc; 8011da177e4SLinus Torvalds as.argv = argv; 8021da177e4SLinus Torvalds 80328f16c20SMicha³ Miros³aw m = alloc_multipath(ti); 8041da177e4SLinus Torvalds if (!m) { 80572d94861SAlasdair G Kergon ti->error = "can't allocate multipath"; 8061da177e4SLinus Torvalds return -EINVAL; 8071da177e4SLinus Torvalds } 8081da177e4SLinus Torvalds 80928f16c20SMicha³ Miros³aw r = parse_features(&as, m); 8101da177e4SLinus Torvalds if (r) 8111da177e4SLinus Torvalds goto bad; 8121da177e4SLinus Torvalds 81328f16c20SMicha³ Miros³aw r = parse_hw_handler(&as, m); 8141da177e4SLinus Torvalds if (r) 8151da177e4SLinus Torvalds goto bad; 8161da177e4SLinus Torvalds 817498f0103SMike Snitzer r = dm_read_arg(_args, &as, &m->nr_priority_groups, &ti->error); 8181da177e4SLinus Torvalds if (r) 8191da177e4SLinus Torvalds goto bad; 8201da177e4SLinus Torvalds 821498f0103SMike Snitzer r = dm_read_arg(_args + 1, &as, &next_pg_num, &ti->error); 8221da177e4SLinus Torvalds if (r) 8231da177e4SLinus Torvalds goto bad; 8241da177e4SLinus Torvalds 825a490a07aSMike Snitzer if ((!m->nr_priority_groups && next_pg_num) || 826a490a07aSMike Snitzer (m->nr_priority_groups && !next_pg_num)) { 827a490a07aSMike Snitzer ti->error = "invalid initial priority group"; 828a490a07aSMike Snitzer r = -EINVAL; 829a490a07aSMike Snitzer goto bad; 830a490a07aSMike Snitzer } 831a490a07aSMike Snitzer 8321da177e4SLinus Torvalds /* parse the priority groups */ 8331da177e4SLinus Torvalds while (as.argc) { 8341da177e4SLinus Torvalds struct priority_group *pg; 8351da177e4SLinus Torvalds 83628f16c20SMicha³ Miros³aw pg = parse_priority_group(&as, m); 83701460f35SBenjamin Marzinski if (IS_ERR(pg)) { 83801460f35SBenjamin Marzinski r = PTR_ERR(pg); 8391da177e4SLinus Torvalds goto bad; 8401da177e4SLinus Torvalds } 8411da177e4SLinus Torvalds 8421da177e4SLinus Torvalds m->nr_valid_paths += pg->nr_pgpaths; 8431da177e4SLinus Torvalds list_add_tail(&pg->list, &m->priority_groups); 8441da177e4SLinus Torvalds pg_count++; 8451da177e4SLinus Torvalds pg->pg_num = pg_count; 8461da177e4SLinus Torvalds if (!--next_pg_num) 8471da177e4SLinus Torvalds m->next_pg = pg; 8481da177e4SLinus Torvalds } 8491da177e4SLinus Torvalds 8501da177e4SLinus Torvalds if (pg_count != m->nr_priority_groups) { 85172d94861SAlasdair G Kergon ti->error = "priority group count mismatch"; 8521da177e4SLinus Torvalds r = -EINVAL; 8531da177e4SLinus Torvalds goto bad; 8541da177e4SLinus Torvalds } 8551da177e4SLinus Torvalds 85655a62eefSAlasdair G Kergon ti->num_flush_bios = 1; 85755a62eefSAlasdair G Kergon ti->num_discard_bios = 1; 858042bcef8SMike Snitzer ti->num_write_same_bios = 1; 8598627921fSMikulas Patocka 8601da177e4SLinus Torvalds return 0; 8611da177e4SLinus Torvalds 8621da177e4SLinus Torvalds bad: 8631da177e4SLinus Torvalds free_multipath(m); 8641da177e4SLinus Torvalds return r; 8651da177e4SLinus Torvalds } 8661da177e4SLinus Torvalds 8672bded7bdSKiyoshi Ueda static void multipath_wait_for_pg_init_completion(struct multipath *m) 8682bded7bdSKiyoshi Ueda { 8692bded7bdSKiyoshi Ueda DECLARE_WAITQUEUE(wait, current); 8702bded7bdSKiyoshi Ueda unsigned long flags; 8712bded7bdSKiyoshi Ueda 8722bded7bdSKiyoshi Ueda add_wait_queue(&m->pg_init_wait, &wait); 8732bded7bdSKiyoshi Ueda 8742bded7bdSKiyoshi Ueda while (1) { 8752bded7bdSKiyoshi Ueda set_current_state(TASK_UNINTERRUPTIBLE); 8762bded7bdSKiyoshi Ueda 8772bded7bdSKiyoshi Ueda spin_lock_irqsave(&m->lock, flags); 8782bded7bdSKiyoshi Ueda if (!m->pg_init_in_progress) { 8792bded7bdSKiyoshi Ueda spin_unlock_irqrestore(&m->lock, flags); 8802bded7bdSKiyoshi Ueda break; 8812bded7bdSKiyoshi Ueda } 8822bded7bdSKiyoshi Ueda spin_unlock_irqrestore(&m->lock, flags); 8832bded7bdSKiyoshi Ueda 8842bded7bdSKiyoshi Ueda io_schedule(); 8852bded7bdSKiyoshi Ueda } 8862bded7bdSKiyoshi Ueda set_current_state(TASK_RUNNING); 8872bded7bdSKiyoshi Ueda 8882bded7bdSKiyoshi Ueda remove_wait_queue(&m->pg_init_wait, &wait); 8892bded7bdSKiyoshi Ueda } 8902bded7bdSKiyoshi Ueda 8912bded7bdSKiyoshi Ueda static void flush_multipath_work(struct multipath *m) 8921da177e4SLinus Torvalds { 893954a73d5SShiva Krishna Merla unsigned long flags; 894954a73d5SShiva Krishna Merla 895954a73d5SShiva Krishna Merla spin_lock_irqsave(&m->lock, flags); 896954a73d5SShiva Krishna Merla m->pg_init_disabled = 1; 897954a73d5SShiva Krishna Merla spin_unlock_irqrestore(&m->lock, flags); 898954a73d5SShiva Krishna Merla 899bab7cfc7SChandra Seetharaman flush_workqueue(kmpath_handlerd); 9002bded7bdSKiyoshi Ueda multipath_wait_for_pg_init_completion(m); 901a044d016SAlasdair G Kergon flush_workqueue(kmultipathd); 90243829731STejun Heo flush_work(&m->trigger_event); 903954a73d5SShiva Krishna Merla 904954a73d5SShiva Krishna Merla spin_lock_irqsave(&m->lock, flags); 905954a73d5SShiva Krishna Merla m->pg_init_disabled = 0; 906954a73d5SShiva Krishna Merla spin_unlock_irqrestore(&m->lock, flags); 9076df400abSKiyoshi Ueda } 9086df400abSKiyoshi Ueda 9096df400abSKiyoshi Ueda static void multipath_dtr(struct dm_target *ti) 9106df400abSKiyoshi Ueda { 9116df400abSKiyoshi Ueda struct multipath *m = ti->private; 9126df400abSKiyoshi Ueda 9132bded7bdSKiyoshi Ueda flush_multipath_work(m); 9141da177e4SLinus Torvalds free_multipath(m); 9151da177e4SLinus Torvalds } 9161da177e4SLinus Torvalds 9171da177e4SLinus Torvalds /* 9181da177e4SLinus Torvalds * Take a path out of use. 9191da177e4SLinus Torvalds */ 9201da177e4SLinus Torvalds static int fail_path(struct pgpath *pgpath) 9211da177e4SLinus Torvalds { 9221da177e4SLinus Torvalds unsigned long flags; 9231da177e4SLinus Torvalds struct multipath *m = pgpath->pg->m; 9241da177e4SLinus Torvalds 9251da177e4SLinus Torvalds spin_lock_irqsave(&m->lock, flags); 9261da177e4SLinus Torvalds 9276680073dSKiyoshi Ueda if (!pgpath->is_active) 9281da177e4SLinus Torvalds goto out; 9291da177e4SLinus Torvalds 93072d94861SAlasdair G Kergon DMWARN("Failing path %s.", pgpath->path.dev->name); 9311da177e4SLinus Torvalds 9321da177e4SLinus Torvalds pgpath->pg->ps.type->fail_path(&pgpath->pg->ps, &pgpath->path); 9336680073dSKiyoshi Ueda pgpath->is_active = 0; 9341da177e4SLinus Torvalds pgpath->fail_count++; 9351da177e4SLinus Torvalds 9361da177e4SLinus Torvalds m->nr_valid_paths--; 9371da177e4SLinus Torvalds 9381da177e4SLinus Torvalds if (pgpath == m->current_pgpath) 9391da177e4SLinus Torvalds m->current_pgpath = NULL; 9401da177e4SLinus Torvalds 941b15546f9SMike Anderson dm_path_uevent(DM_UEVENT_PATH_FAILED, m->ti, 942b15546f9SMike Anderson pgpath->path.dev->name, m->nr_valid_paths); 943b15546f9SMike Anderson 944fe9cf30eSAlasdair G Kergon schedule_work(&m->trigger_event); 9451da177e4SLinus Torvalds 9461da177e4SLinus Torvalds out: 9471da177e4SLinus Torvalds spin_unlock_irqrestore(&m->lock, flags); 9481da177e4SLinus Torvalds 9491da177e4SLinus Torvalds return 0; 9501da177e4SLinus Torvalds } 9511da177e4SLinus Torvalds 9521da177e4SLinus Torvalds /* 9531da177e4SLinus Torvalds * Reinstate a previously-failed path 9541da177e4SLinus Torvalds */ 9551da177e4SLinus Torvalds static int reinstate_path(struct pgpath *pgpath) 9561da177e4SLinus Torvalds { 9571da177e4SLinus Torvalds int r = 0; 9581da177e4SLinus Torvalds unsigned long flags; 9591da177e4SLinus Torvalds struct multipath *m = pgpath->pg->m; 9601da177e4SLinus Torvalds 9611da177e4SLinus Torvalds spin_lock_irqsave(&m->lock, flags); 9621da177e4SLinus Torvalds 9636680073dSKiyoshi Ueda if (pgpath->is_active) 9641da177e4SLinus Torvalds goto out; 9651da177e4SLinus Torvalds 966def052d2SAlasdair G Kergon if (!pgpath->pg->ps.type->reinstate_path) { 9671da177e4SLinus Torvalds DMWARN("Reinstate path not supported by path selector %s", 9681da177e4SLinus Torvalds pgpath->pg->ps.type->name); 9691da177e4SLinus Torvalds r = -EINVAL; 9701da177e4SLinus Torvalds goto out; 9711da177e4SLinus Torvalds } 9721da177e4SLinus Torvalds 9731da177e4SLinus Torvalds r = pgpath->pg->ps.type->reinstate_path(&pgpath->pg->ps, &pgpath->path); 9741da177e4SLinus Torvalds if (r) 9751da177e4SLinus Torvalds goto out; 9761da177e4SLinus Torvalds 9776680073dSKiyoshi Ueda pgpath->is_active = 1; 9781da177e4SLinus Torvalds 979e8099177SHannes Reinecke if (!m->nr_valid_paths++) { 9801da177e4SLinus Torvalds m->current_pgpath = NULL; 9813e9f1be1SHannes Reinecke dm_table_run_md_queue_async(m->ti->table); 982e54f77ddSChandra Seetharaman } else if (m->hw_handler_name && (m->current_pg == pgpath->pg)) { 9834e2d19e4SChandra Seetharaman if (queue_work(kmpath_handlerd, &pgpath->activate_path.work)) 984e54f77ddSChandra Seetharaman m->pg_init_in_progress++; 985e54f77ddSChandra Seetharaman } 9861da177e4SLinus Torvalds 987b15546f9SMike Anderson dm_path_uevent(DM_UEVENT_PATH_REINSTATED, m->ti, 988b15546f9SMike Anderson pgpath->path.dev->name, m->nr_valid_paths); 989b15546f9SMike Anderson 990fe9cf30eSAlasdair G Kergon schedule_work(&m->trigger_event); 9911da177e4SLinus Torvalds 9921da177e4SLinus Torvalds out: 9931da177e4SLinus Torvalds spin_unlock_irqrestore(&m->lock, flags); 9941da177e4SLinus Torvalds 9951da177e4SLinus Torvalds return r; 9961da177e4SLinus Torvalds } 9971da177e4SLinus Torvalds 9981da177e4SLinus Torvalds /* 9991da177e4SLinus Torvalds * Fail or reinstate all paths that match the provided struct dm_dev. 10001da177e4SLinus Torvalds */ 10011da177e4SLinus Torvalds static int action_dev(struct multipath *m, struct dm_dev *dev, 10021da177e4SLinus Torvalds action_fn action) 10031da177e4SLinus Torvalds { 100419040c0bSMike Snitzer int r = -EINVAL; 10051da177e4SLinus Torvalds struct pgpath *pgpath; 10061da177e4SLinus Torvalds struct priority_group *pg; 10071da177e4SLinus Torvalds 10081da177e4SLinus Torvalds list_for_each_entry(pg, &m->priority_groups, list) { 10091da177e4SLinus Torvalds list_for_each_entry(pgpath, &pg->pgpaths, list) { 10101da177e4SLinus Torvalds if (pgpath->path.dev == dev) 10111da177e4SLinus Torvalds r = action(pgpath); 10121da177e4SLinus Torvalds } 10131da177e4SLinus Torvalds } 10141da177e4SLinus Torvalds 10151da177e4SLinus Torvalds return r; 10161da177e4SLinus Torvalds } 10171da177e4SLinus Torvalds 10181da177e4SLinus Torvalds /* 10191da177e4SLinus Torvalds * Temporarily try to avoid having to use the specified PG 10201da177e4SLinus Torvalds */ 10211da177e4SLinus Torvalds static void bypass_pg(struct multipath *m, struct priority_group *pg, 10221da177e4SLinus Torvalds int bypassed) 10231da177e4SLinus Torvalds { 10241da177e4SLinus Torvalds unsigned long flags; 10251da177e4SLinus Torvalds 10261da177e4SLinus Torvalds spin_lock_irqsave(&m->lock, flags); 10271da177e4SLinus Torvalds 10281da177e4SLinus Torvalds pg->bypassed = bypassed; 10291da177e4SLinus Torvalds m->current_pgpath = NULL; 10301da177e4SLinus Torvalds m->current_pg = NULL; 10311da177e4SLinus Torvalds 10321da177e4SLinus Torvalds spin_unlock_irqrestore(&m->lock, flags); 10331da177e4SLinus Torvalds 1034fe9cf30eSAlasdair G Kergon schedule_work(&m->trigger_event); 10351da177e4SLinus Torvalds } 10361da177e4SLinus Torvalds 10371da177e4SLinus Torvalds /* 10381da177e4SLinus Torvalds * Switch to using the specified PG from the next I/O that gets mapped 10391da177e4SLinus Torvalds */ 10401da177e4SLinus Torvalds static int switch_pg_num(struct multipath *m, const char *pgstr) 10411da177e4SLinus Torvalds { 10421da177e4SLinus Torvalds struct priority_group *pg; 10431da177e4SLinus Torvalds unsigned pgnum; 10441da177e4SLinus Torvalds unsigned long flags; 104531998ef1SMikulas Patocka char dummy; 10461da177e4SLinus Torvalds 104731998ef1SMikulas Patocka if (!pgstr || (sscanf(pgstr, "%u%c", &pgnum, &dummy) != 1) || !pgnum || 10481da177e4SLinus Torvalds (pgnum > m->nr_priority_groups)) { 10491da177e4SLinus Torvalds DMWARN("invalid PG number supplied to switch_pg_num"); 10501da177e4SLinus Torvalds return -EINVAL; 10511da177e4SLinus Torvalds } 10521da177e4SLinus Torvalds 10531da177e4SLinus Torvalds spin_lock_irqsave(&m->lock, flags); 10541da177e4SLinus Torvalds list_for_each_entry(pg, &m->priority_groups, list) { 10551da177e4SLinus Torvalds pg->bypassed = 0; 10561da177e4SLinus Torvalds if (--pgnum) 10571da177e4SLinus Torvalds continue; 10581da177e4SLinus Torvalds 10591da177e4SLinus Torvalds m->current_pgpath = NULL; 10601da177e4SLinus Torvalds m->current_pg = NULL; 10611da177e4SLinus Torvalds m->next_pg = pg; 10621da177e4SLinus Torvalds } 10631da177e4SLinus Torvalds spin_unlock_irqrestore(&m->lock, flags); 10641da177e4SLinus Torvalds 1065fe9cf30eSAlasdair G Kergon schedule_work(&m->trigger_event); 10661da177e4SLinus Torvalds return 0; 10671da177e4SLinus Torvalds } 10681da177e4SLinus Torvalds 10691da177e4SLinus Torvalds /* 10701da177e4SLinus Torvalds * Set/clear bypassed status of a PG. 10711da177e4SLinus Torvalds * PGs are numbered upwards from 1 in the order they were declared. 10721da177e4SLinus Torvalds */ 10731da177e4SLinus Torvalds static int bypass_pg_num(struct multipath *m, const char *pgstr, int bypassed) 10741da177e4SLinus Torvalds { 10751da177e4SLinus Torvalds struct priority_group *pg; 10761da177e4SLinus Torvalds unsigned pgnum; 107731998ef1SMikulas Patocka char dummy; 10781da177e4SLinus Torvalds 107931998ef1SMikulas Patocka if (!pgstr || (sscanf(pgstr, "%u%c", &pgnum, &dummy) != 1) || !pgnum || 10801da177e4SLinus Torvalds (pgnum > m->nr_priority_groups)) { 10811da177e4SLinus Torvalds DMWARN("invalid PG number supplied to bypass_pg"); 10821da177e4SLinus Torvalds return -EINVAL; 10831da177e4SLinus Torvalds } 10841da177e4SLinus Torvalds 10851da177e4SLinus Torvalds list_for_each_entry(pg, &m->priority_groups, list) { 10861da177e4SLinus Torvalds if (!--pgnum) 10871da177e4SLinus Torvalds break; 10881da177e4SLinus Torvalds } 10891da177e4SLinus Torvalds 10901da177e4SLinus Torvalds bypass_pg(m, pg, bypassed); 10911da177e4SLinus Torvalds return 0; 10921da177e4SLinus Torvalds } 10931da177e4SLinus Torvalds 10941da177e4SLinus Torvalds /* 1095c9e45581SDave Wysochanski * Should we retry pg_init immediately? 1096c9e45581SDave Wysochanski */ 1097c9e45581SDave Wysochanski static int pg_init_limit_reached(struct multipath *m, struct pgpath *pgpath) 1098c9e45581SDave Wysochanski { 1099c9e45581SDave Wysochanski unsigned long flags; 1100c9e45581SDave Wysochanski int limit_reached = 0; 1101c9e45581SDave Wysochanski 1102c9e45581SDave Wysochanski spin_lock_irqsave(&m->lock, flags); 1103c9e45581SDave Wysochanski 1104954a73d5SShiva Krishna Merla if (m->pg_init_count <= m->pg_init_retries && !m->pg_init_disabled) 1105c9e45581SDave Wysochanski m->pg_init_required = 1; 1106c9e45581SDave Wysochanski else 1107c9e45581SDave Wysochanski limit_reached = 1; 1108c9e45581SDave Wysochanski 1109c9e45581SDave Wysochanski spin_unlock_irqrestore(&m->lock, flags); 1110c9e45581SDave Wysochanski 1111c9e45581SDave Wysochanski return limit_reached; 1112c9e45581SDave Wysochanski } 1113c9e45581SDave Wysochanski 11143ae31f6aSChandra Seetharaman static void pg_init_done(void *data, int errors) 1115cfae5c9bSChandra Seetharaman { 111683c0d5d5SMoger, Babu struct pgpath *pgpath = data; 1117cfae5c9bSChandra Seetharaman struct priority_group *pg = pgpath->pg; 1118cfae5c9bSChandra Seetharaman struct multipath *m = pg->m; 1119cfae5c9bSChandra Seetharaman unsigned long flags; 11204e2d19e4SChandra Seetharaman unsigned delay_retry = 0; 1121cfae5c9bSChandra Seetharaman 1122cfae5c9bSChandra Seetharaman /* device or driver problems */ 1123cfae5c9bSChandra Seetharaman switch (errors) { 1124cfae5c9bSChandra Seetharaman case SCSI_DH_OK: 1125cfae5c9bSChandra Seetharaman break; 1126cfae5c9bSChandra Seetharaman case SCSI_DH_NOSYS: 1127cfae5c9bSChandra Seetharaman if (!m->hw_handler_name) { 1128cfae5c9bSChandra Seetharaman errors = 0; 1129cfae5c9bSChandra Seetharaman break; 1130cfae5c9bSChandra Seetharaman } 1131f7b934c8SMoger, Babu DMERR("Could not failover the device: Handler scsi_dh_%s " 1132f7b934c8SMoger, Babu "Error %d.", m->hw_handler_name, errors); 1133cfae5c9bSChandra Seetharaman /* 1134cfae5c9bSChandra Seetharaman * Fail path for now, so we do not ping pong 1135cfae5c9bSChandra Seetharaman */ 1136cfae5c9bSChandra Seetharaman fail_path(pgpath); 1137cfae5c9bSChandra Seetharaman break; 1138cfae5c9bSChandra Seetharaman case SCSI_DH_DEV_TEMP_BUSY: 1139cfae5c9bSChandra Seetharaman /* 1140cfae5c9bSChandra Seetharaman * Probably doing something like FW upgrade on the 1141cfae5c9bSChandra Seetharaman * controller so try the other pg. 1142cfae5c9bSChandra Seetharaman */ 1143cfae5c9bSChandra Seetharaman bypass_pg(m, pg, 1); 1144cfae5c9bSChandra Seetharaman break; 1145cfae5c9bSChandra Seetharaman case SCSI_DH_RETRY: 11464e2d19e4SChandra Seetharaman /* Wait before retrying. */ 11474e2d19e4SChandra Seetharaman delay_retry = 1; 1148cfae5c9bSChandra Seetharaman case SCSI_DH_IMM_RETRY: 1149cfae5c9bSChandra Seetharaman case SCSI_DH_RES_TEMP_UNAVAIL: 1150cfae5c9bSChandra Seetharaman if (pg_init_limit_reached(m, pgpath)) 1151cfae5c9bSChandra Seetharaman fail_path(pgpath); 1152cfae5c9bSChandra Seetharaman errors = 0; 1153cfae5c9bSChandra Seetharaman break; 1154cfae5c9bSChandra Seetharaman default: 1155cfae5c9bSChandra Seetharaman /* 1156cfae5c9bSChandra Seetharaman * We probably do not want to fail the path for a device 1157cfae5c9bSChandra Seetharaman * error, but this is what the old dm did. In future 1158cfae5c9bSChandra Seetharaman * patches we can do more advanced handling. 1159cfae5c9bSChandra Seetharaman */ 1160cfae5c9bSChandra Seetharaman fail_path(pgpath); 1161cfae5c9bSChandra Seetharaman } 1162cfae5c9bSChandra Seetharaman 1163cfae5c9bSChandra Seetharaman spin_lock_irqsave(&m->lock, flags); 1164cfae5c9bSChandra Seetharaman if (errors) { 1165e54f77ddSChandra Seetharaman if (pgpath == m->current_pgpath) { 1166cfae5c9bSChandra Seetharaman DMERR("Could not failover device. Error %d.", errors); 1167cfae5c9bSChandra Seetharaman m->current_pgpath = NULL; 1168cfae5c9bSChandra Seetharaman m->current_pg = NULL; 1169e54f77ddSChandra Seetharaman } 1170d0259bf0SKiyoshi Ueda } else if (!m->pg_init_required) 1171cfae5c9bSChandra Seetharaman pg->bypassed = 0; 1172cfae5c9bSChandra Seetharaman 1173d0259bf0SKiyoshi Ueda if (--m->pg_init_in_progress) 1174d0259bf0SKiyoshi Ueda /* Activations of other paths are still on going */ 1175d0259bf0SKiyoshi Ueda goto out; 1176d0259bf0SKiyoshi Ueda 11773e9f1be1SHannes Reinecke if (m->pg_init_required) { 11784e2d19e4SChandra Seetharaman m->pg_init_delay_retry = delay_retry; 11793e9f1be1SHannes Reinecke if (__pg_init_all_paths(m)) 11803e9f1be1SHannes Reinecke goto out; 11813e9f1be1SHannes Reinecke } 11823e9f1be1SHannes Reinecke m->queue_io = 0; 1183d0259bf0SKiyoshi Ueda 11842bded7bdSKiyoshi Ueda /* 11852bded7bdSKiyoshi Ueda * Wake up any thread waiting to suspend. 11862bded7bdSKiyoshi Ueda */ 11872bded7bdSKiyoshi Ueda wake_up(&m->pg_init_wait); 11882bded7bdSKiyoshi Ueda 1189d0259bf0SKiyoshi Ueda out: 1190cfae5c9bSChandra Seetharaman spin_unlock_irqrestore(&m->lock, flags); 1191cfae5c9bSChandra Seetharaman } 1192cfae5c9bSChandra Seetharaman 1193bab7cfc7SChandra Seetharaman static void activate_path(struct work_struct *work) 1194bab7cfc7SChandra Seetharaman { 1195e54f77ddSChandra Seetharaman struct pgpath *pgpath = 11964e2d19e4SChandra Seetharaman container_of(work, struct pgpath, activate_path.work); 1197bab7cfc7SChandra Seetharaman 11983a017509SHannes Reinecke if (pgpath->is_active) 11993ae31f6aSChandra Seetharaman scsi_dh_activate(bdev_get_queue(pgpath->path.dev->bdev), 120083c0d5d5SMoger, Babu pg_init_done, pgpath); 12013a017509SHannes Reinecke else 12023a017509SHannes Reinecke pg_init_done(pgpath, SCSI_DH_DEV_OFFLINED); 1203bab7cfc7SChandra Seetharaman } 1204bab7cfc7SChandra Seetharaman 12057e782af5SHannes Reinecke static int noretry_error(int error) 12067e782af5SHannes Reinecke { 12077e782af5SHannes Reinecke switch (error) { 12087e782af5SHannes Reinecke case -EOPNOTSUPP: 12097e782af5SHannes Reinecke case -EREMOTEIO: 12107e782af5SHannes Reinecke case -EILSEQ: 12117e782af5SHannes Reinecke case -ENODATA: 1212cc9d3c38SJun'ichi Nomura case -ENOSPC: 12137e782af5SHannes Reinecke return 1; 12147e782af5SHannes Reinecke } 12157e782af5SHannes Reinecke 12167e782af5SHannes Reinecke /* Anything else could be a path failure, so should be retried */ 12177e782af5SHannes Reinecke return 0; 12187e782af5SHannes Reinecke } 12197e782af5SHannes Reinecke 12201da177e4SLinus Torvalds /* 12211da177e4SLinus Torvalds * end_io handling 12221da177e4SLinus Torvalds */ 1223f40c67f0SKiyoshi Ueda static int do_end_io(struct multipath *m, struct request *clone, 1224028867acSAlasdair G Kergon int error, struct dm_mpath_io *mpio) 12251da177e4SLinus Torvalds { 1226f40c67f0SKiyoshi Ueda /* 1227f40c67f0SKiyoshi Ueda * We don't queue any clone request inside the multipath target 1228f40c67f0SKiyoshi Ueda * during end I/O handling, since those clone requests don't have 1229f40c67f0SKiyoshi Ueda * bio clones. If we queue them inside the multipath target, 1230f40c67f0SKiyoshi Ueda * we need to make bio clones, that requires memory allocation. 1231f40c67f0SKiyoshi Ueda * (See drivers/md/dm.c:end_clone_bio() about why the clone requests 1232f40c67f0SKiyoshi Ueda * don't have bio clones.) 1233f40c67f0SKiyoshi Ueda * Instead of queueing the clone request here, we queue the original 1234f40c67f0SKiyoshi Ueda * request into dm core, which will remake a clone request and 1235f40c67f0SKiyoshi Ueda * clone bios for it and resubmit it later. 1236f40c67f0SKiyoshi Ueda */ 1237f40c67f0SKiyoshi Ueda int r = DM_ENDIO_REQUEUE; 1238640eb3b0SStefan Bader unsigned long flags; 12391da177e4SLinus Torvalds 1240f40c67f0SKiyoshi Ueda if (!error && !clone->errors) 12411da177e4SLinus Torvalds return 0; /* I/O complete */ 12421da177e4SLinus Torvalds 1243f84cb8a4SMike Snitzer if (noretry_error(error)) { 1244f84cb8a4SMike Snitzer if ((clone->cmd_flags & REQ_WRITE_SAME) && 1245f84cb8a4SMike Snitzer !clone->q->limits.max_write_same_sectors) { 1246f84cb8a4SMike Snitzer struct queue_limits *limits; 1247f84cb8a4SMike Snitzer 1248f84cb8a4SMike Snitzer /* device doesn't really support WRITE SAME, disable it */ 1249f84cb8a4SMike Snitzer limits = dm_get_queue_limits(dm_table_get_md(m->ti->table)); 1250f84cb8a4SMike Snitzer limits->max_write_same_sectors = 0; 1251f84cb8a4SMike Snitzer } 1252959eb4e5SMike Snitzer return error; 1253f84cb8a4SMike Snitzer } 1254959eb4e5SMike Snitzer 1255cfae5c9bSChandra Seetharaman if (mpio->pgpath) 12561da177e4SLinus Torvalds fail_path(mpio->pgpath); 12571da177e4SLinus Torvalds 1258640eb3b0SStefan Bader spin_lock_irqsave(&m->lock, flags); 1259751b2a7dSHannes Reinecke if (!m->nr_valid_paths) { 1260751b2a7dSHannes Reinecke if (!m->queue_if_no_path) { 1261751b2a7dSHannes Reinecke if (!__must_push_back(m)) 1262f40c67f0SKiyoshi Ueda r = -EIO; 1263751b2a7dSHannes Reinecke } else { 1264751b2a7dSHannes Reinecke if (error == -EBADE) 1265751b2a7dSHannes Reinecke r = error; 1266751b2a7dSHannes Reinecke } 1267751b2a7dSHannes Reinecke } 1268640eb3b0SStefan Bader spin_unlock_irqrestore(&m->lock, flags); 12691da177e4SLinus Torvalds 1270f40c67f0SKiyoshi Ueda return r; 12711da177e4SLinus Torvalds } 12721da177e4SLinus Torvalds 1273f40c67f0SKiyoshi Ueda static int multipath_end_io(struct dm_target *ti, struct request *clone, 12741da177e4SLinus Torvalds int error, union map_info *map_context) 12751da177e4SLinus Torvalds { 1276028867acSAlasdair G Kergon struct multipath *m = ti->private; 1277028867acSAlasdair G Kergon struct dm_mpath_io *mpio = map_context->ptr; 1278a71a261fSWei Yongjun struct pgpath *pgpath; 12791da177e4SLinus Torvalds struct path_selector *ps; 12801da177e4SLinus Torvalds int r; 12811da177e4SLinus Torvalds 1282466891f9SJun'ichi Nomura BUG_ON(!mpio); 1283466891f9SJun'ichi Nomura 1284f40c67f0SKiyoshi Ueda r = do_end_io(m, clone, error, mpio); 1285a71a261fSWei Yongjun pgpath = mpio->pgpath; 12861da177e4SLinus Torvalds if (pgpath) { 12871da177e4SLinus Torvalds ps = &pgpath->pg->ps; 12881da177e4SLinus Torvalds if (ps->type->end_io) 128902ab823fSKiyoshi Ueda ps->type->end_io(ps, &pgpath->path, mpio->nr_bytes); 12901da177e4SLinus Torvalds } 1291466891f9SJun'ichi Nomura clear_mapinfo(m, map_context); 12921da177e4SLinus Torvalds 12931da177e4SLinus Torvalds return r; 12941da177e4SLinus Torvalds } 12951da177e4SLinus Torvalds 12961da177e4SLinus Torvalds /* 12971da177e4SLinus Torvalds * Suspend can't complete until all the I/O is processed so if 1298436d4108SAlasdair G Kergon * the last path fails we must error any remaining I/O. 1299436d4108SAlasdair G Kergon * Note that if the freeze_bdev fails while suspending, the 1300436d4108SAlasdair G Kergon * queue_if_no_path state is lost - userspace should reset it. 13011da177e4SLinus Torvalds */ 13021da177e4SLinus Torvalds static void multipath_presuspend(struct dm_target *ti) 13031da177e4SLinus Torvalds { 13041da177e4SLinus Torvalds struct multipath *m = (struct multipath *) ti->private; 13051da177e4SLinus Torvalds 1306485ef69eSAlasdair G Kergon queue_if_no_path(m, 0, 1); 13071da177e4SLinus Torvalds } 13081da177e4SLinus Torvalds 13096df400abSKiyoshi Ueda static void multipath_postsuspend(struct dm_target *ti) 13106df400abSKiyoshi Ueda { 13116380f26fSMike Anderson struct multipath *m = ti->private; 13126380f26fSMike Anderson 13136380f26fSMike Anderson mutex_lock(&m->work_mutex); 13142bded7bdSKiyoshi Ueda flush_multipath_work(m); 13156380f26fSMike Anderson mutex_unlock(&m->work_mutex); 13166df400abSKiyoshi Ueda } 13176df400abSKiyoshi Ueda 1318436d4108SAlasdair G Kergon /* 1319436d4108SAlasdair G Kergon * Restore the queue_if_no_path setting. 1320436d4108SAlasdair G Kergon */ 13211da177e4SLinus Torvalds static void multipath_resume(struct dm_target *ti) 13221da177e4SLinus Torvalds { 13231da177e4SLinus Torvalds struct multipath *m = (struct multipath *) ti->private; 13241da177e4SLinus Torvalds unsigned long flags; 13251da177e4SLinus Torvalds 13261da177e4SLinus Torvalds spin_lock_irqsave(&m->lock, flags); 1327436d4108SAlasdair G Kergon m->queue_if_no_path = m->saved_queue_if_no_path; 13281da177e4SLinus Torvalds spin_unlock_irqrestore(&m->lock, flags); 13291da177e4SLinus Torvalds } 13301da177e4SLinus Torvalds 13311da177e4SLinus Torvalds /* 13321da177e4SLinus Torvalds * Info output has the following format: 13331da177e4SLinus Torvalds * num_multipath_feature_args [multipath_feature_args]* 13341da177e4SLinus Torvalds * num_handler_status_args [handler_status_args]* 13351da177e4SLinus Torvalds * num_groups init_group_number 13361da177e4SLinus Torvalds * [A|D|E num_ps_status_args [ps_status_args]* 13371da177e4SLinus Torvalds * num_paths num_selector_args 13381da177e4SLinus Torvalds * [path_dev A|F fail_count [selector_args]* ]+ ]+ 13391da177e4SLinus Torvalds * 13401da177e4SLinus Torvalds * Table output has the following format (identical to the constructor string): 13411da177e4SLinus Torvalds * num_feature_args [features_args]* 13421da177e4SLinus Torvalds * num_handler_args hw_handler [hw_handler_args]* 13431da177e4SLinus Torvalds * num_groups init_group_number 13441da177e4SLinus Torvalds * [priority selector-name num_ps_args [ps_args]* 13451da177e4SLinus Torvalds * num_paths num_selector_args [path_dev [selector_args]* ]+ ]+ 13461da177e4SLinus Torvalds */ 1347fd7c092eSMikulas Patocka static void multipath_status(struct dm_target *ti, status_type_t type, 13481f4e0ff0SAlasdair G Kergon unsigned status_flags, char *result, unsigned maxlen) 13491da177e4SLinus Torvalds { 13501da177e4SLinus Torvalds int sz = 0; 13511da177e4SLinus Torvalds unsigned long flags; 13521da177e4SLinus Torvalds struct multipath *m = (struct multipath *) ti->private; 13531da177e4SLinus Torvalds struct priority_group *pg; 13541da177e4SLinus Torvalds struct pgpath *p; 13551da177e4SLinus Torvalds unsigned pg_num; 13561da177e4SLinus Torvalds char state; 13571da177e4SLinus Torvalds 13581da177e4SLinus Torvalds spin_lock_irqsave(&m->lock, flags); 13591da177e4SLinus Torvalds 13601da177e4SLinus Torvalds /* Features */ 13611da177e4SLinus Torvalds if (type == STATUSTYPE_INFO) 1362e8099177SHannes Reinecke DMEMIT("2 %u %u ", m->queue_io, m->pg_init_count); 1363c9e45581SDave Wysochanski else { 1364c9e45581SDave Wysochanski DMEMIT("%u ", m->queue_if_no_path + 13654e2d19e4SChandra Seetharaman (m->pg_init_retries > 0) * 2 + 1366a58a935dSMike Snitzer (m->pg_init_delay_msecs != DM_PG_INIT_DELAY_DEFAULT) * 2 + 1367a58a935dSMike Snitzer m->retain_attached_hw_handler); 1368c9e45581SDave Wysochanski if (m->queue_if_no_path) 1369c9e45581SDave Wysochanski DMEMIT("queue_if_no_path "); 1370c9e45581SDave Wysochanski if (m->pg_init_retries) 1371c9e45581SDave Wysochanski DMEMIT("pg_init_retries %u ", m->pg_init_retries); 13724e2d19e4SChandra Seetharaman if (m->pg_init_delay_msecs != DM_PG_INIT_DELAY_DEFAULT) 13734e2d19e4SChandra Seetharaman DMEMIT("pg_init_delay_msecs %u ", m->pg_init_delay_msecs); 1374a58a935dSMike Snitzer if (m->retain_attached_hw_handler) 1375a58a935dSMike Snitzer DMEMIT("retain_attached_hw_handler "); 1376c9e45581SDave Wysochanski } 13771da177e4SLinus Torvalds 1378cfae5c9bSChandra Seetharaman if (!m->hw_handler_name || type == STATUSTYPE_INFO) 13791da177e4SLinus Torvalds DMEMIT("0 "); 13801da177e4SLinus Torvalds else 1381cfae5c9bSChandra Seetharaman DMEMIT("1 %s ", m->hw_handler_name); 13821da177e4SLinus Torvalds 13831da177e4SLinus Torvalds DMEMIT("%u ", m->nr_priority_groups); 13841da177e4SLinus Torvalds 13851da177e4SLinus Torvalds if (m->next_pg) 13861da177e4SLinus Torvalds pg_num = m->next_pg->pg_num; 13871da177e4SLinus Torvalds else if (m->current_pg) 13881da177e4SLinus Torvalds pg_num = m->current_pg->pg_num; 13891da177e4SLinus Torvalds else 1390a490a07aSMike Snitzer pg_num = (m->nr_priority_groups ? 1 : 0); 13911da177e4SLinus Torvalds 13921da177e4SLinus Torvalds DMEMIT("%u ", pg_num); 13931da177e4SLinus Torvalds 13941da177e4SLinus Torvalds switch (type) { 13951da177e4SLinus Torvalds case STATUSTYPE_INFO: 13961da177e4SLinus Torvalds list_for_each_entry(pg, &m->priority_groups, list) { 13971da177e4SLinus Torvalds if (pg->bypassed) 13981da177e4SLinus Torvalds state = 'D'; /* Disabled */ 13991da177e4SLinus Torvalds else if (pg == m->current_pg) 14001da177e4SLinus Torvalds state = 'A'; /* Currently Active */ 14011da177e4SLinus Torvalds else 14021da177e4SLinus Torvalds state = 'E'; /* Enabled */ 14031da177e4SLinus Torvalds 14041da177e4SLinus Torvalds DMEMIT("%c ", state); 14051da177e4SLinus Torvalds 14061da177e4SLinus Torvalds if (pg->ps.type->status) 14071da177e4SLinus Torvalds sz += pg->ps.type->status(&pg->ps, NULL, type, 14081da177e4SLinus Torvalds result + sz, 14091da177e4SLinus Torvalds maxlen - sz); 14101da177e4SLinus Torvalds else 14111da177e4SLinus Torvalds DMEMIT("0 "); 14121da177e4SLinus Torvalds 14131da177e4SLinus Torvalds DMEMIT("%u %u ", pg->nr_pgpaths, 14141da177e4SLinus Torvalds pg->ps.type->info_args); 14151da177e4SLinus Torvalds 14161da177e4SLinus Torvalds list_for_each_entry(p, &pg->pgpaths, list) { 14171da177e4SLinus Torvalds DMEMIT("%s %s %u ", p->path.dev->name, 14186680073dSKiyoshi Ueda p->is_active ? "A" : "F", 14191da177e4SLinus Torvalds p->fail_count); 14201da177e4SLinus Torvalds if (pg->ps.type->status) 14211da177e4SLinus Torvalds sz += pg->ps.type->status(&pg->ps, 14221da177e4SLinus Torvalds &p->path, type, result + sz, 14231da177e4SLinus Torvalds maxlen - sz); 14241da177e4SLinus Torvalds } 14251da177e4SLinus Torvalds } 14261da177e4SLinus Torvalds break; 14271da177e4SLinus Torvalds 14281da177e4SLinus Torvalds case STATUSTYPE_TABLE: 14291da177e4SLinus Torvalds list_for_each_entry(pg, &m->priority_groups, list) { 14301da177e4SLinus Torvalds DMEMIT("%s ", pg->ps.type->name); 14311da177e4SLinus Torvalds 14321da177e4SLinus Torvalds if (pg->ps.type->status) 14331da177e4SLinus Torvalds sz += pg->ps.type->status(&pg->ps, NULL, type, 14341da177e4SLinus Torvalds result + sz, 14351da177e4SLinus Torvalds maxlen - sz); 14361da177e4SLinus Torvalds else 14371da177e4SLinus Torvalds DMEMIT("0 "); 14381da177e4SLinus Torvalds 14391da177e4SLinus Torvalds DMEMIT("%u %u ", pg->nr_pgpaths, 14401da177e4SLinus Torvalds pg->ps.type->table_args); 14411da177e4SLinus Torvalds 14421da177e4SLinus Torvalds list_for_each_entry(p, &pg->pgpaths, list) { 14431da177e4SLinus Torvalds DMEMIT("%s ", p->path.dev->name); 14441da177e4SLinus Torvalds if (pg->ps.type->status) 14451da177e4SLinus Torvalds sz += pg->ps.type->status(&pg->ps, 14461da177e4SLinus Torvalds &p->path, type, result + sz, 14471da177e4SLinus Torvalds maxlen - sz); 14481da177e4SLinus Torvalds } 14491da177e4SLinus Torvalds } 14501da177e4SLinus Torvalds break; 14511da177e4SLinus Torvalds } 14521da177e4SLinus Torvalds 14531da177e4SLinus Torvalds spin_unlock_irqrestore(&m->lock, flags); 14541da177e4SLinus Torvalds } 14551da177e4SLinus Torvalds 14561da177e4SLinus Torvalds static int multipath_message(struct dm_target *ti, unsigned argc, char **argv) 14571da177e4SLinus Torvalds { 14586380f26fSMike Anderson int r = -EINVAL; 14591da177e4SLinus Torvalds struct dm_dev *dev; 14601da177e4SLinus Torvalds struct multipath *m = (struct multipath *) ti->private; 14611da177e4SLinus Torvalds action_fn action; 14621da177e4SLinus Torvalds 14636380f26fSMike Anderson mutex_lock(&m->work_mutex); 14646380f26fSMike Anderson 1465c2f3d24bSKiyoshi Ueda if (dm_suspended(ti)) { 1466c2f3d24bSKiyoshi Ueda r = -EBUSY; 1467c2f3d24bSKiyoshi Ueda goto out; 1468c2f3d24bSKiyoshi Ueda } 1469c2f3d24bSKiyoshi Ueda 14701da177e4SLinus Torvalds if (argc == 1) { 1471498f0103SMike Snitzer if (!strcasecmp(argv[0], "queue_if_no_path")) { 14726380f26fSMike Anderson r = queue_if_no_path(m, 1, 0); 14736380f26fSMike Anderson goto out; 1474498f0103SMike Snitzer } else if (!strcasecmp(argv[0], "fail_if_no_path")) { 14756380f26fSMike Anderson r = queue_if_no_path(m, 0, 0); 14766380f26fSMike Anderson goto out; 14776380f26fSMike Anderson } 14781da177e4SLinus Torvalds } 14791da177e4SLinus Torvalds 14806380f26fSMike Anderson if (argc != 2) { 1481*a356e426SJose Castillo DMWARN("Invalid multipath message arguments. Expected 2 arguments, got %d.", argc); 14826380f26fSMike Anderson goto out; 14836380f26fSMike Anderson } 14841da177e4SLinus Torvalds 1485498f0103SMike Snitzer if (!strcasecmp(argv[0], "disable_group")) { 14866380f26fSMike Anderson r = bypass_pg_num(m, argv[1], 1); 14876380f26fSMike Anderson goto out; 1488498f0103SMike Snitzer } else if (!strcasecmp(argv[0], "enable_group")) { 14896380f26fSMike Anderson r = bypass_pg_num(m, argv[1], 0); 14906380f26fSMike Anderson goto out; 1491498f0103SMike Snitzer } else if (!strcasecmp(argv[0], "switch_group")) { 14926380f26fSMike Anderson r = switch_pg_num(m, argv[1]); 14936380f26fSMike Anderson goto out; 1494498f0103SMike Snitzer } else if (!strcasecmp(argv[0], "reinstate_path")) 14951da177e4SLinus Torvalds action = reinstate_path; 1496498f0103SMike Snitzer else if (!strcasecmp(argv[0], "fail_path")) 14971da177e4SLinus Torvalds action = fail_path; 14986380f26fSMike Anderson else { 1499*a356e426SJose Castillo DMWARN("Unrecognised multipath message received: %s", argv[0]); 15006380f26fSMike Anderson goto out; 15016380f26fSMike Anderson } 15021da177e4SLinus Torvalds 15038215d6ecSNikanth Karthikesan r = dm_get_device(ti, argv[1], dm_table_get_mode(ti->table), &dev); 15041da177e4SLinus Torvalds if (r) { 150572d94861SAlasdair G Kergon DMWARN("message: error getting device %s", 15061da177e4SLinus Torvalds argv[1]); 15076380f26fSMike Anderson goto out; 15081da177e4SLinus Torvalds } 15091da177e4SLinus Torvalds 15101da177e4SLinus Torvalds r = action_dev(m, dev, action); 15111da177e4SLinus Torvalds 15121da177e4SLinus Torvalds dm_put_device(ti, dev); 15131da177e4SLinus Torvalds 15146380f26fSMike Anderson out: 15156380f26fSMike Anderson mutex_unlock(&m->work_mutex); 15161da177e4SLinus Torvalds return r; 15171da177e4SLinus Torvalds } 15181da177e4SLinus Torvalds 1519647b3d00SAl Viro static int multipath_ioctl(struct dm_target *ti, unsigned int cmd, 15209af4aa30SMilan Broz unsigned long arg) 15219af4aa30SMilan Broz { 152235991652SMikulas Patocka struct multipath *m = ti->private; 15237ba10aa6SMike Snitzer struct pgpath *pgpath; 152435991652SMikulas Patocka struct block_device *bdev; 152535991652SMikulas Patocka fmode_t mode; 15269af4aa30SMilan Broz unsigned long flags; 152735991652SMikulas Patocka int r; 152835991652SMikulas Patocka 152935991652SMikulas Patocka bdev = NULL; 153035991652SMikulas Patocka mode = 0; 153135991652SMikulas Patocka r = 0; 15329af4aa30SMilan Broz 15339af4aa30SMilan Broz spin_lock_irqsave(&m->lock, flags); 15349af4aa30SMilan Broz 15359af4aa30SMilan Broz if (!m->current_pgpath) 153602ab823fSKiyoshi Ueda __choose_pgpath(m, 0); 15379af4aa30SMilan Broz 15387ba10aa6SMike Snitzer pgpath = m->current_pgpath; 15397ba10aa6SMike Snitzer 15407ba10aa6SMike Snitzer if (pgpath) { 15417ba10aa6SMike Snitzer bdev = pgpath->path.dev->bdev; 15427ba10aa6SMike Snitzer mode = pgpath->path.dev->mode; 1543e90dae1fSMilan Broz } 15449af4aa30SMilan Broz 15457ba10aa6SMike Snitzer if ((pgpath && m->queue_io) || (!pgpath && m->queue_if_no_path)) 15466c182cd8SHannes Reinecke r = -ENOTCONN; 15479af4aa30SMilan Broz else if (!bdev) 15489af4aa30SMilan Broz r = -EIO; 15499af4aa30SMilan Broz 15509af4aa30SMilan Broz spin_unlock_irqrestore(&m->lock, flags); 15519af4aa30SMilan Broz 1552ec8013beSPaolo Bonzini /* 1553ec8013beSPaolo Bonzini * Only pass ioctls through if the device sizes match exactly. 1554ec8013beSPaolo Bonzini */ 1555a1989b33SHannes Reinecke if (!bdev || ti->len != i_size_read(bdev->bd_inode) >> SECTOR_SHIFT) { 1556a1989b33SHannes Reinecke int err = scsi_verify_blk_ioctl(NULL, cmd); 1557a1989b33SHannes Reinecke if (err) 1558a1989b33SHannes Reinecke r = err; 1559a1989b33SHannes Reinecke } 1560ec8013beSPaolo Bonzini 15613e9f1be1SHannes Reinecke if (r == -ENOTCONN && !fatal_signal_pending(current)) { 15623e9f1be1SHannes Reinecke spin_lock_irqsave(&m->lock, flags); 15633e9f1be1SHannes Reinecke if (!m->current_pg) { 15643e9f1be1SHannes Reinecke /* Path status changed, redo selection */ 15653e9f1be1SHannes Reinecke __choose_pgpath(m, 0); 15663e9f1be1SHannes Reinecke } 15673e9f1be1SHannes Reinecke if (m->pg_init_required) 15683e9f1be1SHannes Reinecke __pg_init_all_paths(m); 15693e9f1be1SHannes Reinecke spin_unlock_irqrestore(&m->lock, flags); 15703e9f1be1SHannes Reinecke dm_table_run_md_queue_async(m->ti->table); 15713e9f1be1SHannes Reinecke } 157235991652SMikulas Patocka 1573633a08b8SAl Viro return r ? : __blkdev_driver_ioctl(bdev, mode, cmd, arg); 15749af4aa30SMilan Broz } 15759af4aa30SMilan Broz 1576af4874e0SMike Snitzer static int multipath_iterate_devices(struct dm_target *ti, 1577af4874e0SMike Snitzer iterate_devices_callout_fn fn, void *data) 1578af4874e0SMike Snitzer { 1579af4874e0SMike Snitzer struct multipath *m = ti->private; 1580af4874e0SMike Snitzer struct priority_group *pg; 1581af4874e0SMike Snitzer struct pgpath *p; 1582af4874e0SMike Snitzer int ret = 0; 1583af4874e0SMike Snitzer 1584af4874e0SMike Snitzer list_for_each_entry(pg, &m->priority_groups, list) { 1585af4874e0SMike Snitzer list_for_each_entry(p, &pg->pgpaths, list) { 15865dea271bSMike Snitzer ret = fn(ti, p->path.dev, ti->begin, ti->len, data); 1587af4874e0SMike Snitzer if (ret) 1588af4874e0SMike Snitzer goto out; 1589af4874e0SMike Snitzer } 1590af4874e0SMike Snitzer } 1591af4874e0SMike Snitzer 1592af4874e0SMike Snitzer out: 1593af4874e0SMike Snitzer return ret; 1594af4874e0SMike Snitzer } 1595af4874e0SMike Snitzer 1596f40c67f0SKiyoshi Ueda static int __pgpath_busy(struct pgpath *pgpath) 1597f40c67f0SKiyoshi Ueda { 1598f40c67f0SKiyoshi Ueda struct request_queue *q = bdev_get_queue(pgpath->path.dev->bdev); 1599f40c67f0SKiyoshi Ueda 1600f40c67f0SKiyoshi Ueda return dm_underlying_device_busy(q); 1601f40c67f0SKiyoshi Ueda } 1602f40c67f0SKiyoshi Ueda 1603f40c67f0SKiyoshi Ueda /* 1604f40c67f0SKiyoshi Ueda * We return "busy", only when we can map I/Os but underlying devices 1605f40c67f0SKiyoshi Ueda * are busy (so even if we map I/Os now, the I/Os will wait on 1606f40c67f0SKiyoshi Ueda * the underlying queue). 1607f40c67f0SKiyoshi Ueda * In other words, if we want to kill I/Os or queue them inside us 1608f40c67f0SKiyoshi Ueda * due to map unavailability, we don't return "busy". Otherwise, 1609f40c67f0SKiyoshi Ueda * dm core won't give us the I/Os and we can't do what we want. 1610f40c67f0SKiyoshi Ueda */ 1611f40c67f0SKiyoshi Ueda static int multipath_busy(struct dm_target *ti) 1612f40c67f0SKiyoshi Ueda { 1613f40c67f0SKiyoshi Ueda int busy = 0, has_active = 0; 1614f40c67f0SKiyoshi Ueda struct multipath *m = ti->private; 1615f40c67f0SKiyoshi Ueda struct priority_group *pg; 1616f40c67f0SKiyoshi Ueda struct pgpath *pgpath; 1617f40c67f0SKiyoshi Ueda unsigned long flags; 1618f40c67f0SKiyoshi Ueda 1619f40c67f0SKiyoshi Ueda spin_lock_irqsave(&m->lock, flags); 1620f40c67f0SKiyoshi Ueda 1621b63349a7SHannes Reinecke /* pg_init in progress, requeue until done */ 1622e8099177SHannes Reinecke if (!pg_ready(m)) { 1623b63349a7SHannes Reinecke busy = 1; 1624b63349a7SHannes Reinecke goto out; 1625b63349a7SHannes Reinecke } 1626f40c67f0SKiyoshi Ueda /* Guess which priority_group will be used at next mapping time */ 1627f40c67f0SKiyoshi Ueda if (unlikely(!m->current_pgpath && m->next_pg)) 1628f40c67f0SKiyoshi Ueda pg = m->next_pg; 1629f40c67f0SKiyoshi Ueda else if (likely(m->current_pg)) 1630f40c67f0SKiyoshi Ueda pg = m->current_pg; 1631f40c67f0SKiyoshi Ueda else 1632f40c67f0SKiyoshi Ueda /* 1633f40c67f0SKiyoshi Ueda * We don't know which pg will be used at next mapping time. 1634f40c67f0SKiyoshi Ueda * We don't call __choose_pgpath() here to avoid to trigger 1635f40c67f0SKiyoshi Ueda * pg_init just by busy checking. 1636f40c67f0SKiyoshi Ueda * So we don't know whether underlying devices we will be using 1637f40c67f0SKiyoshi Ueda * at next mapping time are busy or not. Just try mapping. 1638f40c67f0SKiyoshi Ueda */ 1639f40c67f0SKiyoshi Ueda goto out; 1640f40c67f0SKiyoshi Ueda 1641f40c67f0SKiyoshi Ueda /* 1642f40c67f0SKiyoshi Ueda * If there is one non-busy active path at least, the path selector 1643f40c67f0SKiyoshi Ueda * will be able to select it. So we consider such a pg as not busy. 1644f40c67f0SKiyoshi Ueda */ 1645f40c67f0SKiyoshi Ueda busy = 1; 1646f40c67f0SKiyoshi Ueda list_for_each_entry(pgpath, &pg->pgpaths, list) 1647f40c67f0SKiyoshi Ueda if (pgpath->is_active) { 1648f40c67f0SKiyoshi Ueda has_active = 1; 1649f40c67f0SKiyoshi Ueda 1650f40c67f0SKiyoshi Ueda if (!__pgpath_busy(pgpath)) { 1651f40c67f0SKiyoshi Ueda busy = 0; 1652f40c67f0SKiyoshi Ueda break; 1653f40c67f0SKiyoshi Ueda } 1654f40c67f0SKiyoshi Ueda } 1655f40c67f0SKiyoshi Ueda 1656f40c67f0SKiyoshi Ueda if (!has_active) 1657f40c67f0SKiyoshi Ueda /* 1658f40c67f0SKiyoshi Ueda * No active path in this pg, so this pg won't be used and 1659f40c67f0SKiyoshi Ueda * the current_pg will be changed at next mapping time. 1660f40c67f0SKiyoshi Ueda * We need to try mapping to determine it. 1661f40c67f0SKiyoshi Ueda */ 1662f40c67f0SKiyoshi Ueda busy = 0; 1663f40c67f0SKiyoshi Ueda 1664f40c67f0SKiyoshi Ueda out: 1665f40c67f0SKiyoshi Ueda spin_unlock_irqrestore(&m->lock, flags); 1666f40c67f0SKiyoshi Ueda 1667f40c67f0SKiyoshi Ueda return busy; 1668f40c67f0SKiyoshi Ueda } 1669f40c67f0SKiyoshi Ueda 16701da177e4SLinus Torvalds /*----------------------------------------------------------------- 16711da177e4SLinus Torvalds * Module setup 16721da177e4SLinus Torvalds *---------------------------------------------------------------*/ 16731da177e4SLinus Torvalds static struct target_type multipath_target = { 16741da177e4SLinus Torvalds .name = "multipath", 1675e8099177SHannes Reinecke .version = {1, 7, 0}, 16761da177e4SLinus Torvalds .module = THIS_MODULE, 16771da177e4SLinus Torvalds .ctr = multipath_ctr, 16781da177e4SLinus Torvalds .dtr = multipath_dtr, 1679f40c67f0SKiyoshi Ueda .map_rq = multipath_map, 1680f40c67f0SKiyoshi Ueda .rq_end_io = multipath_end_io, 16811da177e4SLinus Torvalds .presuspend = multipath_presuspend, 16826df400abSKiyoshi Ueda .postsuspend = multipath_postsuspend, 16831da177e4SLinus Torvalds .resume = multipath_resume, 16841da177e4SLinus Torvalds .status = multipath_status, 16851da177e4SLinus Torvalds .message = multipath_message, 16869af4aa30SMilan Broz .ioctl = multipath_ioctl, 1687af4874e0SMike Snitzer .iterate_devices = multipath_iterate_devices, 1688f40c67f0SKiyoshi Ueda .busy = multipath_busy, 16891da177e4SLinus Torvalds }; 16901da177e4SLinus Torvalds 16911da177e4SLinus Torvalds static int __init dm_multipath_init(void) 16921da177e4SLinus Torvalds { 16931da177e4SLinus Torvalds int r; 16941da177e4SLinus Torvalds 16951da177e4SLinus Torvalds /* allocate a slab for the dm_ios */ 1696028867acSAlasdair G Kergon _mpio_cache = KMEM_CACHE(dm_mpath_io, 0); 16971da177e4SLinus Torvalds if (!_mpio_cache) 16981da177e4SLinus Torvalds return -ENOMEM; 16991da177e4SLinus Torvalds 17001da177e4SLinus Torvalds r = dm_register_target(&multipath_target); 17011da177e4SLinus Torvalds if (r < 0) { 17020cd33124SAlasdair G Kergon DMERR("register failed %d", r); 17031da177e4SLinus Torvalds kmem_cache_destroy(_mpio_cache); 17041da177e4SLinus Torvalds return -EINVAL; 17051da177e4SLinus Torvalds } 17061da177e4SLinus Torvalds 17074d4d66abSTejun Heo kmultipathd = alloc_workqueue("kmpathd", WQ_MEM_RECLAIM, 0); 1708c557308eSAlasdair G Kergon if (!kmultipathd) { 17090cd33124SAlasdair G Kergon DMERR("failed to create workqueue kmpathd"); 1710c557308eSAlasdair G Kergon dm_unregister_target(&multipath_target); 1711c557308eSAlasdair G Kergon kmem_cache_destroy(_mpio_cache); 1712c557308eSAlasdair G Kergon return -ENOMEM; 1713c557308eSAlasdair G Kergon } 1714c557308eSAlasdair G Kergon 1715bab7cfc7SChandra Seetharaman /* 1716bab7cfc7SChandra Seetharaman * A separate workqueue is used to handle the device handlers 1717bab7cfc7SChandra Seetharaman * to avoid overloading existing workqueue. Overloading the 1718bab7cfc7SChandra Seetharaman * old workqueue would also create a bottleneck in the 1719bab7cfc7SChandra Seetharaman * path of the storage hardware device activation. 1720bab7cfc7SChandra Seetharaman */ 17214d4d66abSTejun Heo kmpath_handlerd = alloc_ordered_workqueue("kmpath_handlerd", 17224d4d66abSTejun Heo WQ_MEM_RECLAIM); 1723bab7cfc7SChandra Seetharaman if (!kmpath_handlerd) { 1724bab7cfc7SChandra Seetharaman DMERR("failed to create workqueue kmpath_handlerd"); 1725bab7cfc7SChandra Seetharaman destroy_workqueue(kmultipathd); 1726bab7cfc7SChandra Seetharaman dm_unregister_target(&multipath_target); 1727bab7cfc7SChandra Seetharaman kmem_cache_destroy(_mpio_cache); 1728bab7cfc7SChandra Seetharaman return -ENOMEM; 1729bab7cfc7SChandra Seetharaman } 1730bab7cfc7SChandra Seetharaman 173172d94861SAlasdair G Kergon DMINFO("version %u.%u.%u loaded", 17321da177e4SLinus Torvalds multipath_target.version[0], multipath_target.version[1], 17331da177e4SLinus Torvalds multipath_target.version[2]); 17341da177e4SLinus Torvalds 17351da177e4SLinus Torvalds return r; 17361da177e4SLinus Torvalds } 17371da177e4SLinus Torvalds 17381da177e4SLinus Torvalds static void __exit dm_multipath_exit(void) 17391da177e4SLinus Torvalds { 1740bab7cfc7SChandra Seetharaman destroy_workqueue(kmpath_handlerd); 1741c557308eSAlasdair G Kergon destroy_workqueue(kmultipathd); 1742c557308eSAlasdair G Kergon 174310d3bd09SMikulas Patocka dm_unregister_target(&multipath_target); 17441da177e4SLinus Torvalds kmem_cache_destroy(_mpio_cache); 17451da177e4SLinus Torvalds } 17461da177e4SLinus Torvalds 17471da177e4SLinus Torvalds module_init(dm_multipath_init); 17481da177e4SLinus Torvalds module_exit(dm_multipath_exit); 17491da177e4SLinus Torvalds 17501da177e4SLinus Torvalds MODULE_DESCRIPTION(DM_NAME " multipath target"); 17511da177e4SLinus Torvalds MODULE_AUTHOR("Sistina Software <dm-devel@redhat.com>"); 17521da177e4SLinus Torvalds MODULE_LICENSE("GPL"); 1753