1 /* 2 * 3 * Copyright (c) 2009, Microsoft Corporation. 4 * 5 * This program is free software; you can redistribute it and/or modify it 6 * under the terms and conditions of the GNU General Public License, 7 * version 2, as published by the Free Software Foundation. 8 * 9 * This program is distributed in the hope it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 12 * more details. 13 * 14 * You should have received a copy of the GNU General Public License along with 15 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple 16 * Place - Suite 330, Boston, MA 02111-1307 USA. 17 * 18 * Authors: 19 * Haiyang Zhang <haiyangz@microsoft.com> 20 * Hank Janssen <hjanssen@microsoft.com> 21 * K. Y. Srinivasan <kys@microsoft.com> 22 * 23 */ 24 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 25 26 #include <linux/kernel.h> 27 #include <linux/mm.h> 28 #include <linux/hyperv.h> 29 #include <linux/uio.h> 30 31 #include "hyperv_vmbus.h" 32 33 void hv_begin_read(struct hv_ring_buffer_info *rbi) 34 { 35 rbi->ring_buffer->interrupt_mask = 1; 36 mb(); 37 } 38 39 u32 hv_end_read(struct hv_ring_buffer_info *rbi) 40 { 41 u32 read; 42 u32 write; 43 44 rbi->ring_buffer->interrupt_mask = 0; 45 mb(); 46 47 /* 48 * Now check to see if the ring buffer is still empty. 49 * If it is not, we raced and we need to process new 50 * incoming messages. 51 */ 52 hv_get_ringbuffer_availbytes(rbi, &read, &write); 53 54 return read; 55 } 56 57 /* 58 * When we write to the ring buffer, check if the host needs to 59 * be signaled. Here is the details of this protocol: 60 * 61 * 1. The host guarantees that while it is draining the 62 * ring buffer, it will set the interrupt_mask to 63 * indicate it does not need to be interrupted when 64 * new data is placed. 65 * 66 * 2. The host guarantees that it will completely drain 67 * the ring buffer before exiting the read loop. Further, 68 * once the ring buffer is empty, it will clear the 69 * interrupt_mask and re-check to see if new data has 70 * arrived. 71 */ 72 73 static bool hv_need_to_signal(u32 old_write, struct hv_ring_buffer_info *rbi) 74 { 75 mb(); 76 if (rbi->ring_buffer->interrupt_mask) 77 return false; 78 79 /* check interrupt_mask before read_index */ 80 rmb(); 81 /* 82 * This is the only case we need to signal when the 83 * ring transitions from being empty to non-empty. 84 */ 85 if (old_write == rbi->ring_buffer->read_index) 86 return true; 87 88 return false; 89 } 90 91 /* 92 * To optimize the flow management on the send-side, 93 * when the sender is blocked because of lack of 94 * sufficient space in the ring buffer, potential the 95 * consumer of the ring buffer can signal the producer. 96 * This is controlled by the following parameters: 97 * 98 * 1. pending_send_sz: This is the size in bytes that the 99 * producer is trying to send. 100 * 2. The feature bit feat_pending_send_sz set to indicate if 101 * the consumer of the ring will signal when the ring 102 * state transitions from being full to a state where 103 * there is room for the producer to send the pending packet. 104 */ 105 106 static bool hv_need_to_signal_on_read(u32 prev_write_sz, 107 struct hv_ring_buffer_info *rbi) 108 { 109 u32 cur_write_sz; 110 u32 r_size; 111 u32 write_loc = rbi->ring_buffer->write_index; 112 u32 read_loc = rbi->ring_buffer->read_index; 113 u32 pending_sz = rbi->ring_buffer->pending_send_sz; 114 115 /* If the other end is not blocked on write don't bother. */ 116 if (pending_sz == 0) 117 return false; 118 119 r_size = rbi->ring_datasize; 120 cur_write_sz = write_loc >= read_loc ? r_size - (write_loc - read_loc) : 121 read_loc - write_loc; 122 123 if ((prev_write_sz < pending_sz) && (cur_write_sz >= pending_sz)) 124 return true; 125 126 return false; 127 } 128 129 /* Get the next write location for the specified ring buffer. */ 130 static inline u32 131 hv_get_next_write_location(struct hv_ring_buffer_info *ring_info) 132 { 133 u32 next = ring_info->ring_buffer->write_index; 134 135 return next; 136 } 137 138 /* Set the next write location for the specified ring buffer. */ 139 static inline void 140 hv_set_next_write_location(struct hv_ring_buffer_info *ring_info, 141 u32 next_write_location) 142 { 143 ring_info->ring_buffer->write_index = next_write_location; 144 } 145 146 /* Get the next read location for the specified ring buffer. */ 147 static inline u32 148 hv_get_next_read_location(struct hv_ring_buffer_info *ring_info) 149 { 150 u32 next = ring_info->ring_buffer->read_index; 151 152 return next; 153 } 154 155 /* 156 * Get the next read location + offset for the specified ring buffer. 157 * This allows the caller to skip. 158 */ 159 static inline u32 160 hv_get_next_readlocation_withoffset(struct hv_ring_buffer_info *ring_info, 161 u32 offset) 162 { 163 u32 next = ring_info->ring_buffer->read_index; 164 165 next += offset; 166 next %= ring_info->ring_datasize; 167 168 return next; 169 } 170 171 /* Set the next read location for the specified ring buffer. */ 172 static inline void 173 hv_set_next_read_location(struct hv_ring_buffer_info *ring_info, 174 u32 next_read_location) 175 { 176 ring_info->ring_buffer->read_index = next_read_location; 177 } 178 179 180 /* Get the start of the ring buffer. */ 181 static inline void * 182 hv_get_ring_buffer(struct hv_ring_buffer_info *ring_info) 183 { 184 return (void *)ring_info->ring_buffer->buffer; 185 } 186 187 188 /* Get the size of the ring buffer. */ 189 static inline u32 190 hv_get_ring_buffersize(struct hv_ring_buffer_info *ring_info) 191 { 192 return ring_info->ring_datasize; 193 } 194 195 /* Get the read and write indices as u64 of the specified ring buffer. */ 196 static inline u64 197 hv_get_ring_bufferindices(struct hv_ring_buffer_info *ring_info) 198 { 199 return (u64)ring_info->ring_buffer->write_index << 32; 200 } 201 202 /* 203 * Helper routine to copy to source from ring buffer. 204 * Assume there is enough room. Handles wrap-around in src case only!! 205 */ 206 static u32 hv_copyfrom_ringbuffer( 207 struct hv_ring_buffer_info *ring_info, 208 void *dest, 209 u32 destlen, 210 u32 start_read_offset) 211 { 212 void *ring_buffer = hv_get_ring_buffer(ring_info); 213 u32 ring_buffer_size = hv_get_ring_buffersize(ring_info); 214 215 u32 frag_len; 216 217 /* wrap-around detected at the src */ 218 if (destlen > ring_buffer_size - start_read_offset) { 219 frag_len = ring_buffer_size - start_read_offset; 220 221 memcpy(dest, ring_buffer + start_read_offset, frag_len); 222 memcpy(dest + frag_len, ring_buffer, destlen - frag_len); 223 } else 224 225 memcpy(dest, ring_buffer + start_read_offset, destlen); 226 227 228 start_read_offset += destlen; 229 start_read_offset %= ring_buffer_size; 230 231 return start_read_offset; 232 } 233 234 235 /* 236 * Helper routine to copy from source to ring buffer. 237 * Assume there is enough room. Handles wrap-around in dest case only!! 238 */ 239 static u32 hv_copyto_ringbuffer( 240 struct hv_ring_buffer_info *ring_info, 241 u32 start_write_offset, 242 void *src, 243 u32 srclen) 244 { 245 void *ring_buffer = hv_get_ring_buffer(ring_info); 246 u32 ring_buffer_size = hv_get_ring_buffersize(ring_info); 247 u32 frag_len; 248 249 /* wrap-around detected! */ 250 if (srclen > ring_buffer_size - start_write_offset) { 251 frag_len = ring_buffer_size - start_write_offset; 252 memcpy(ring_buffer + start_write_offset, src, frag_len); 253 memcpy(ring_buffer, src + frag_len, srclen - frag_len); 254 } else 255 memcpy(ring_buffer + start_write_offset, src, srclen); 256 257 start_write_offset += srclen; 258 start_write_offset %= ring_buffer_size; 259 260 return start_write_offset; 261 } 262 263 /* Get various debug metrics for the specified ring buffer. */ 264 void hv_ringbuffer_get_debuginfo(struct hv_ring_buffer_info *ring_info, 265 struct hv_ring_buffer_debug_info *debug_info) 266 { 267 u32 bytes_avail_towrite; 268 u32 bytes_avail_toread; 269 270 if (ring_info->ring_buffer) { 271 hv_get_ringbuffer_availbytes(ring_info, 272 &bytes_avail_toread, 273 &bytes_avail_towrite); 274 275 debug_info->bytes_avail_toread = bytes_avail_toread; 276 debug_info->bytes_avail_towrite = bytes_avail_towrite; 277 debug_info->current_read_index = 278 ring_info->ring_buffer->read_index; 279 debug_info->current_write_index = 280 ring_info->ring_buffer->write_index; 281 debug_info->current_interrupt_mask = 282 ring_info->ring_buffer->interrupt_mask; 283 } 284 } 285 286 /* Initialize the ring buffer. */ 287 int hv_ringbuffer_init(struct hv_ring_buffer_info *ring_info, 288 void *buffer, u32 buflen) 289 { 290 if (sizeof(struct hv_ring_buffer) != PAGE_SIZE) 291 return -EINVAL; 292 293 memset(ring_info, 0, sizeof(struct hv_ring_buffer_info)); 294 295 ring_info->ring_buffer = (struct hv_ring_buffer *)buffer; 296 ring_info->ring_buffer->read_index = 297 ring_info->ring_buffer->write_index = 0; 298 299 /* Set the feature bit for enabling flow control. */ 300 ring_info->ring_buffer->feature_bits.value = 1; 301 302 ring_info->ring_size = buflen; 303 ring_info->ring_datasize = buflen - sizeof(struct hv_ring_buffer); 304 305 spin_lock_init(&ring_info->ring_lock); 306 307 return 0; 308 } 309 310 /* Cleanup the ring buffer. */ 311 void hv_ringbuffer_cleanup(struct hv_ring_buffer_info *ring_info) 312 { 313 } 314 315 /* Write to the ring buffer. */ 316 int hv_ringbuffer_write(struct hv_ring_buffer_info *outring_info, 317 struct kvec *kv_list, u32 kv_count, bool *signal, bool lock) 318 { 319 int i = 0; 320 u32 bytes_avail_towrite; 321 u32 bytes_avail_toread; 322 u32 totalbytes_towrite = 0; 323 324 u32 next_write_location; 325 u32 old_write; 326 u64 prev_indices = 0; 327 unsigned long flags = 0; 328 329 for (i = 0; i < kv_count; i++) 330 totalbytes_towrite += kv_list[i].iov_len; 331 332 totalbytes_towrite += sizeof(u64); 333 334 if (lock) 335 spin_lock_irqsave(&outring_info->ring_lock, flags); 336 337 hv_get_ringbuffer_availbytes(outring_info, 338 &bytes_avail_toread, 339 &bytes_avail_towrite); 340 341 /* 342 * If there is only room for the packet, assume it is full. 343 * Otherwise, the next time around, we think the ring buffer 344 * is empty since the read index == write index. 345 */ 346 if (bytes_avail_towrite <= totalbytes_towrite) { 347 if (lock) 348 spin_unlock_irqrestore(&outring_info->ring_lock, flags); 349 return -EAGAIN; 350 } 351 352 /* Write to the ring buffer */ 353 next_write_location = hv_get_next_write_location(outring_info); 354 355 old_write = next_write_location; 356 357 for (i = 0; i < kv_count; i++) { 358 next_write_location = hv_copyto_ringbuffer(outring_info, 359 next_write_location, 360 kv_list[i].iov_base, 361 kv_list[i].iov_len); 362 } 363 364 /* Set previous packet start */ 365 prev_indices = hv_get_ring_bufferindices(outring_info); 366 367 next_write_location = hv_copyto_ringbuffer(outring_info, 368 next_write_location, 369 &prev_indices, 370 sizeof(u64)); 371 372 /* Issue a full memory barrier before updating the write index */ 373 mb(); 374 375 /* Now, update the write location */ 376 hv_set_next_write_location(outring_info, next_write_location); 377 378 379 if (lock) 380 spin_unlock_irqrestore(&outring_info->ring_lock, flags); 381 382 *signal = hv_need_to_signal(old_write, outring_info); 383 return 0; 384 } 385 386 int hv_ringbuffer_read(struct hv_ring_buffer_info *inring_info, 387 void *buffer, u32 buflen, u32 *buffer_actual_len, 388 u64 *requestid, bool *signal, bool raw) 389 { 390 u32 bytes_avail_towrite; 391 u32 bytes_avail_toread; 392 u32 next_read_location = 0; 393 u64 prev_indices = 0; 394 struct vmpacket_descriptor desc; 395 u32 offset; 396 u32 packetlen; 397 int ret = 0; 398 399 if (buflen <= 0) 400 return -EINVAL; 401 402 403 *buffer_actual_len = 0; 404 *requestid = 0; 405 406 hv_get_ringbuffer_availbytes(inring_info, 407 &bytes_avail_toread, 408 &bytes_avail_towrite); 409 410 /* Make sure there is something to read */ 411 if (bytes_avail_toread < sizeof(desc)) { 412 /* 413 * No error is set when there is even no header, drivers are 414 * supposed to analyze buffer_actual_len. 415 */ 416 return ret; 417 } 418 419 next_read_location = hv_get_next_read_location(inring_info); 420 next_read_location = hv_copyfrom_ringbuffer(inring_info, &desc, 421 sizeof(desc), 422 next_read_location); 423 424 offset = raw ? 0 : (desc.offset8 << 3); 425 packetlen = (desc.len8 << 3) - offset; 426 *buffer_actual_len = packetlen; 427 *requestid = desc.trans_id; 428 429 if (bytes_avail_toread < packetlen + offset) 430 return -EAGAIN; 431 432 if (packetlen > buflen) 433 return -ENOBUFS; 434 435 next_read_location = 436 hv_get_next_readlocation_withoffset(inring_info, offset); 437 438 next_read_location = hv_copyfrom_ringbuffer(inring_info, 439 buffer, 440 packetlen, 441 next_read_location); 442 443 next_read_location = hv_copyfrom_ringbuffer(inring_info, 444 &prev_indices, 445 sizeof(u64), 446 next_read_location); 447 448 /* 449 * Make sure all reads are done before we update the read index since 450 * the writer may start writing to the read area once the read index 451 * is updated. 452 */ 453 mb(); 454 455 /* Update the read index */ 456 hv_set_next_read_location(inring_info, next_read_location); 457 458 *signal = hv_need_to_signal_on_read(bytes_avail_towrite, inring_info); 459 460 return ret; 461 } 462