1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* 3 * Copyright (c) 2018, Mellanox Technologies inc. All rights reserved. 4 */ 5 6 #include <linux/dim.h> 7 8 /* 9 * Net DIM profiles: 10 * There are different set of profiles for each CQ period mode. 11 * There are different set of profiles for RX/TX CQs. 12 * Each profile size must be of NET_DIM_PARAMS_NUM_PROFILES 13 */ 14 #define NET_DIM_PARAMS_NUM_PROFILES 5 15 #define NET_DIM_DEFAULT_RX_CQ_PKTS_FROM_EQE 256 16 #define NET_DIM_DEFAULT_TX_CQ_PKTS_FROM_EQE 128 17 #define NET_DIM_DEF_PROFILE_CQE 1 18 #define NET_DIM_DEF_PROFILE_EQE 1 19 20 #define NET_DIM_RX_EQE_PROFILES { \ 21 {.usec = 1, .pkts = NET_DIM_DEFAULT_RX_CQ_PKTS_FROM_EQE,}, \ 22 {.usec = 8, .pkts = NET_DIM_DEFAULT_RX_CQ_PKTS_FROM_EQE,}, \ 23 {.usec = 64, .pkts = NET_DIM_DEFAULT_RX_CQ_PKTS_FROM_EQE,}, \ 24 {.usec = 128, .pkts = NET_DIM_DEFAULT_RX_CQ_PKTS_FROM_EQE,}, \ 25 {.usec = 256, .pkts = NET_DIM_DEFAULT_RX_CQ_PKTS_FROM_EQE,} \ 26 } 27 28 #define NET_DIM_RX_CQE_PROFILES { \ 29 {.usec = 2, .pkts = 256,}, \ 30 {.usec = 8, .pkts = 128,}, \ 31 {.usec = 16, .pkts = 64,}, \ 32 {.usec = 32, .pkts = 64,}, \ 33 {.usec = 64, .pkts = 64,} \ 34 } 35 36 #define NET_DIM_TX_EQE_PROFILES { \ 37 {.usec = 1, .pkts = NET_DIM_DEFAULT_TX_CQ_PKTS_FROM_EQE,}, \ 38 {.usec = 8, .pkts = NET_DIM_DEFAULT_TX_CQ_PKTS_FROM_EQE,}, \ 39 {.usec = 32, .pkts = NET_DIM_DEFAULT_TX_CQ_PKTS_FROM_EQE,}, \ 40 {.usec = 64, .pkts = NET_DIM_DEFAULT_TX_CQ_PKTS_FROM_EQE,}, \ 41 {.usec = 128, .pkts = NET_DIM_DEFAULT_TX_CQ_PKTS_FROM_EQE,} \ 42 } 43 44 #define NET_DIM_TX_CQE_PROFILES { \ 45 {.usec = 5, .pkts = 128,}, \ 46 {.usec = 8, .pkts = 64,}, \ 47 {.usec = 16, .pkts = 32,}, \ 48 {.usec = 32, .pkts = 32,}, \ 49 {.usec = 64, .pkts = 32,} \ 50 } 51 52 static const struct dim_cq_moder 53 rx_profile[DIM_CQ_PERIOD_NUM_MODES][NET_DIM_PARAMS_NUM_PROFILES] = { 54 NET_DIM_RX_EQE_PROFILES, 55 NET_DIM_RX_CQE_PROFILES, 56 }; 57 58 static const struct dim_cq_moder 59 tx_profile[DIM_CQ_PERIOD_NUM_MODES][NET_DIM_PARAMS_NUM_PROFILES] = { 60 NET_DIM_TX_EQE_PROFILES, 61 NET_DIM_TX_CQE_PROFILES, 62 }; 63 64 struct dim_cq_moder 65 net_dim_get_rx_moderation(u8 cq_period_mode, int ix) 66 { 67 struct dim_cq_moder cq_moder = rx_profile[cq_period_mode][ix]; 68 69 cq_moder.cq_period_mode = cq_period_mode; 70 return cq_moder; 71 } 72 EXPORT_SYMBOL(net_dim_get_rx_moderation); 73 74 struct dim_cq_moder 75 net_dim_get_def_rx_moderation(u8 cq_period_mode) 76 { 77 u8 profile_ix = cq_period_mode == DIM_CQ_PERIOD_MODE_START_FROM_CQE ? 78 NET_DIM_DEF_PROFILE_CQE : NET_DIM_DEF_PROFILE_EQE; 79 80 return net_dim_get_rx_moderation(cq_period_mode, profile_ix); 81 } 82 EXPORT_SYMBOL(net_dim_get_def_rx_moderation); 83 84 struct dim_cq_moder 85 net_dim_get_tx_moderation(u8 cq_period_mode, int ix) 86 { 87 struct dim_cq_moder cq_moder = tx_profile[cq_period_mode][ix]; 88 89 cq_moder.cq_period_mode = cq_period_mode; 90 return cq_moder; 91 } 92 EXPORT_SYMBOL(net_dim_get_tx_moderation); 93 94 struct dim_cq_moder 95 net_dim_get_def_tx_moderation(u8 cq_period_mode) 96 { 97 u8 profile_ix = cq_period_mode == DIM_CQ_PERIOD_MODE_START_FROM_CQE ? 98 NET_DIM_DEF_PROFILE_CQE : NET_DIM_DEF_PROFILE_EQE; 99 100 return net_dim_get_tx_moderation(cq_period_mode, profile_ix); 101 } 102 EXPORT_SYMBOL(net_dim_get_def_tx_moderation); 103 104 static int net_dim_step(struct dim *dim) 105 { 106 if (dim->tired == (NET_DIM_PARAMS_NUM_PROFILES * 2)) 107 return DIM_TOO_TIRED; 108 109 switch (dim->tune_state) { 110 case DIM_PARKING_ON_TOP: 111 case DIM_PARKING_TIRED: 112 break; 113 case DIM_GOING_RIGHT: 114 if (dim->profile_ix == (NET_DIM_PARAMS_NUM_PROFILES - 1)) 115 return DIM_ON_EDGE; 116 dim->profile_ix++; 117 dim->steps_right++; 118 break; 119 case DIM_GOING_LEFT: 120 if (dim->profile_ix == 0) 121 return DIM_ON_EDGE; 122 dim->profile_ix--; 123 dim->steps_left++; 124 break; 125 } 126 127 dim->tired++; 128 return DIM_STEPPED; 129 } 130 131 static void net_dim_exit_parking(struct dim *dim) 132 { 133 dim->tune_state = dim->profile_ix ? DIM_GOING_LEFT : DIM_GOING_RIGHT; 134 net_dim_step(dim); 135 } 136 137 static int net_dim_stats_compare(struct dim_stats *curr, 138 struct dim_stats *prev) 139 { 140 if (!prev->bpms) 141 return curr->bpms ? DIM_STATS_BETTER : DIM_STATS_SAME; 142 143 if (IS_SIGNIFICANT_DIFF(curr->bpms, prev->bpms)) 144 return (curr->bpms > prev->bpms) ? DIM_STATS_BETTER : 145 DIM_STATS_WORSE; 146 147 if (!prev->ppms) 148 return curr->ppms ? DIM_STATS_BETTER : 149 DIM_STATS_SAME; 150 151 if (IS_SIGNIFICANT_DIFF(curr->ppms, prev->ppms)) 152 return (curr->ppms > prev->ppms) ? DIM_STATS_BETTER : 153 DIM_STATS_WORSE; 154 155 if (!prev->epms) 156 return DIM_STATS_SAME; 157 158 if (IS_SIGNIFICANT_DIFF(curr->epms, prev->epms)) 159 return (curr->epms < prev->epms) ? DIM_STATS_BETTER : 160 DIM_STATS_WORSE; 161 162 return DIM_STATS_SAME; 163 } 164 165 static bool net_dim_decision(struct dim_stats *curr_stats, struct dim *dim) 166 { 167 int prev_state = dim->tune_state; 168 int prev_ix = dim->profile_ix; 169 int stats_res; 170 int step_res; 171 172 switch (dim->tune_state) { 173 case DIM_PARKING_ON_TOP: 174 stats_res = net_dim_stats_compare(curr_stats, 175 &dim->prev_stats); 176 if (stats_res != DIM_STATS_SAME) 177 net_dim_exit_parking(dim); 178 break; 179 180 case DIM_PARKING_TIRED: 181 dim->tired--; 182 if (!dim->tired) 183 net_dim_exit_parking(dim); 184 break; 185 186 case DIM_GOING_RIGHT: 187 case DIM_GOING_LEFT: 188 stats_res = net_dim_stats_compare(curr_stats, 189 &dim->prev_stats); 190 if (stats_res != DIM_STATS_BETTER) 191 dim_turn(dim); 192 193 if (dim_on_top(dim)) { 194 dim_park_on_top(dim); 195 break; 196 } 197 198 step_res = net_dim_step(dim); 199 switch (step_res) { 200 case DIM_ON_EDGE: 201 dim_park_on_top(dim); 202 break; 203 case DIM_TOO_TIRED: 204 dim_park_tired(dim); 205 break; 206 } 207 208 break; 209 } 210 211 if (prev_state != DIM_PARKING_ON_TOP || 212 dim->tune_state != DIM_PARKING_ON_TOP) 213 dim->prev_stats = *curr_stats; 214 215 return dim->profile_ix != prev_ix; 216 } 217 218 void net_dim(struct dim *dim, struct dim_sample end_sample) 219 { 220 struct dim_stats curr_stats; 221 u16 nevents; 222 223 switch (dim->state) { 224 case DIM_MEASURE_IN_PROGRESS: 225 nevents = BIT_GAP(BITS_PER_TYPE(u16), 226 end_sample.event_ctr, 227 dim->start_sample.event_ctr); 228 if (nevents < DIM_NEVENTS) 229 break; 230 dim_calc_stats(&dim->start_sample, &end_sample, &curr_stats); 231 if (net_dim_decision(&curr_stats, dim)) { 232 dim->state = DIM_APPLY_NEW_PROFILE; 233 schedule_work(&dim->work); 234 break; 235 } 236 fallthrough; 237 case DIM_START_MEASURE: 238 dim_update_sample(end_sample.event_ctr, end_sample.pkt_ctr, 239 end_sample.byte_ctr, &dim->start_sample); 240 dim->state = DIM_MEASURE_IN_PROGRESS; 241 break; 242 case DIM_APPLY_NEW_PROFILE: 243 break; 244 } 245 } 246 EXPORT_SYMBOL(net_dim); 247