1 /* 2 * Copyright (c) 2016 Oracle. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 #ifndef _RDS_IB_MR_H 33 #define _RDS_IB_MR_H 34 35 #include <linux/kernel.h> 36 37 #include "rds.h" 38 #include "ib.h" 39 40 #define RDS_MR_1M_POOL_SIZE (8192 / 2) 41 #define RDS_MR_1M_MSG_SIZE 256 42 #define RDS_MR_8K_MSG_SIZE 2 43 #define RDS_MR_8K_SCALE (256 / (RDS_MR_8K_MSG_SIZE + 1)) 44 #define RDS_MR_8K_POOL_SIZE (RDS_MR_8K_SCALE * (8192 / 2)) 45 46 struct rds_ib_fmr { 47 struct ib_fmr *fmr; 48 }; 49 50 enum rds_ib_fr_state { 51 FRMR_IS_FREE, /* mr invalidated & ready for use */ 52 FRMR_IS_INUSE, /* mr is in use or used & can be invalidated */ 53 FRMR_IS_STALE, /* Stale MR and needs to be dropped */ 54 }; 55 56 struct rds_ib_frmr { 57 struct ib_mr *mr; 58 enum rds_ib_fr_state fr_state; 59 bool fr_inv; 60 wait_queue_head_t fr_inv_done; 61 bool fr_reg; 62 wait_queue_head_t fr_reg_done; 63 struct ib_send_wr fr_wr; 64 unsigned int dma_npages; 65 unsigned int sg_byte_len; 66 }; 67 68 /* This is stored as mr->r_trans_private. */ 69 struct rds_ib_mr { 70 struct delayed_work work; 71 struct rds_ib_device *device; 72 struct rds_ib_mr_pool *pool; 73 struct rds_ib_connection *ic; 74 75 struct llist_node llnode; 76 77 /* unmap_list is for freeing */ 78 struct list_head unmap_list; 79 unsigned int remap_count; 80 81 struct scatterlist *sg; 82 unsigned int sg_len; 83 int sg_dma_len; 84 85 u8 odp:1; 86 union { 87 struct rds_ib_fmr fmr; 88 struct rds_ib_frmr frmr; 89 struct ib_mr *mr; 90 } u; 91 }; 92 93 /* Our own little MR pool */ 94 struct rds_ib_mr_pool { 95 unsigned int pool_type; 96 struct mutex flush_lock; /* serialize fmr invalidate */ 97 struct delayed_work flush_worker; /* flush worker */ 98 99 atomic_t item_count; /* total # of MRs */ 100 atomic_t dirty_count; /* # dirty of MRs */ 101 102 struct llist_head drop_list; /* MRs not reached max_maps */ 103 struct llist_head free_list; /* unused MRs */ 104 struct llist_head clean_list; /* unused & unmapped MRs */ 105 wait_queue_head_t flush_wait; 106 spinlock_t clean_lock; /* "clean_list" concurrency */ 107 108 atomic_t free_pinned; /* memory pinned by free MRs */ 109 unsigned long max_items; 110 unsigned long max_items_soft; 111 unsigned long max_free_pinned; 112 struct ib_fmr_attr fmr_attr; 113 bool use_fastreg; 114 }; 115 116 extern struct workqueue_struct *rds_ib_mr_wq; 117 extern bool prefer_frmr; 118 119 struct rds_ib_mr_pool *rds_ib_create_mr_pool(struct rds_ib_device *rds_dev, 120 int npages); 121 void rds_ib_get_mr_info(struct rds_ib_device *rds_ibdev, 122 struct rds_info_rdma_connection *iinfo); 123 void rds6_ib_get_mr_info(struct rds_ib_device *rds_ibdev, 124 struct rds6_info_rdma_connection *iinfo6); 125 void rds_ib_destroy_mr_pool(struct rds_ib_mr_pool *); 126 void *rds_ib_get_mr(struct scatterlist *sg, unsigned long nents, 127 struct rds_sock *rs, u32 *key_ret, 128 struct rds_connection *conn, u64 start, u64 length, 129 int need_odp); 130 void rds_ib_sync_mr(void *trans_private, int dir); 131 void rds_ib_free_mr(void *trans_private, int invalidate); 132 void rds_ib_flush_mrs(void); 133 int rds_ib_mr_init(void); 134 void rds_ib_mr_exit(void); 135 u32 rds_ib_get_lkey(void *trans_private); 136 137 void __rds_ib_teardown_mr(struct rds_ib_mr *); 138 void rds_ib_teardown_mr(struct rds_ib_mr *); 139 struct rds_ib_mr *rds_ib_alloc_fmr(struct rds_ib_device *, int); 140 struct rds_ib_mr *rds_ib_reuse_mr(struct rds_ib_mr_pool *); 141 int rds_ib_flush_mr_pool(struct rds_ib_mr_pool *, int, struct rds_ib_mr **); 142 struct rds_ib_mr *rds_ib_reg_fmr(struct rds_ib_device *, struct scatterlist *, 143 unsigned long, u32 *); 144 struct rds_ib_mr *rds_ib_try_reuse_ibmr(struct rds_ib_mr_pool *); 145 void rds_ib_unreg_fmr(struct list_head *, unsigned int *, 146 unsigned long *, unsigned int); 147 void rds_ib_free_fmr_list(struct rds_ib_mr *); 148 struct rds_ib_mr *rds_ib_reg_frmr(struct rds_ib_device *rds_ibdev, 149 struct rds_ib_connection *ic, 150 struct scatterlist *sg, 151 unsigned long nents, u32 *key); 152 void rds_ib_unreg_frmr(struct list_head *list, unsigned int *nfreed, 153 unsigned long *unpinned, unsigned int goal); 154 void rds_ib_free_frmr_list(struct rds_ib_mr *); 155 #endif 156