1 /*
2  * Copyright(c) 2015, 2016 Intel Corporation.
3  *
4  * This file is provided under a dual BSD/GPLv2 license.  When using or
5  * redistributing this file, you may do so under either license.
6  *
7  * GPL LICENSE SUMMARY
8  *
9  * This program is free software; you can redistribute it and/or modify
10  * it under the terms of version 2 of the GNU General Public License as
11  * published by the Free Software Foundation.
12  *
13  * This program is distributed in the hope that it will be useful, but
14  * WITHOUT ANY WARRANTY; without even the implied warranty of
15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
16  * General Public License for more details.
17  *
18  * BSD LICENSE
19  *
20  * Redistribution and use in source and binary forms, with or without
21  * modification, are permitted provided that the following conditions
22  * are met:
23  *
24  *  - Redistributions of source code must retain the above copyright
25  *    notice, this list of conditions and the following disclaimer.
26  *  - Redistributions in binary form must reproduce the above copyright
27  *    notice, this list of conditions and the following disclaimer in
28  *    the documentation and/or other materials provided with the
29  *    distribution.
30  *  - Neither the name of Intel Corporation nor the names of its
31  *    contributors may be used to endorse or promote products derived
32  *    from this software without specific prior written permission.
33  *
34  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
35  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
36  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
37  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
38  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
39  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
40  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
41  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
42  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
43  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
44  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
45  *
46  */
47 
48 #include "hfi.h"
49 
50 /* additive distance between non-SOP and SOP space */
51 #define SOP_DISTANCE (TXE_PIO_SIZE / 2)
52 #define PIO_BLOCK_MASK (PIO_BLOCK_SIZE - 1)
53 /* number of QUADWORDs in a block */
54 #define PIO_BLOCK_QWS (PIO_BLOCK_SIZE / sizeof(u64))
55 
56 /**
57  * pio_copy - copy data block to MMIO space
58  * @pbuf: a number of blocks allocated within a PIO send context
59  * @pbc: PBC to send
60  * @from: source, must be 8 byte aligned
61  * @count: number of DWORD (32-bit) quantities to copy from source
62  *
63  * Copy data from source to PIO Send Buffer memory, 8 bytes at a time.
64  * Must always write full BLOCK_SIZE bytes blocks.  The first block must
65  * be written to the corresponding SOP=1 address.
66  *
67  * Known:
68  * o pbuf->start always starts on a block boundary
69  * o pbuf can wrap only at a block boundary
70  */
71 void pio_copy(struct hfi1_devdata *dd, struct pio_buf *pbuf, u64 pbc,
72 	      const void *from, size_t count)
73 {
74 	void __iomem *dest = pbuf->start + SOP_DISTANCE;
75 	void __iomem *send = dest + PIO_BLOCK_SIZE;
76 	void __iomem *dend;			/* 8-byte data end */
77 
78 	/* write the PBC */
79 	writeq(pbc, dest);
80 	dest += sizeof(u64);
81 
82 	/* calculate where the QWORD data ends - in SOP=1 space */
83 	dend = dest + ((count >> 1) * sizeof(u64));
84 
85 	if (dend < send) {
86 		/*
87 		 * all QWORD data is within the SOP block, does *not*
88 		 * reach the end of the SOP block
89 		 */
90 
91 		while (dest < dend) {
92 			writeq(*(u64 *)from, dest);
93 			from += sizeof(u64);
94 			dest += sizeof(u64);
95 		}
96 		/*
97 		 * No boundary checks are needed here:
98 		 * 0. We're not on the SOP block boundary
99 		 * 1. The possible DWORD dangle will still be within
100 		 *    the SOP block
101 		 * 2. We cannot wrap except on a block boundary.
102 		 */
103 	} else {
104 		/* QWORD data extends _to_ or beyond the SOP block */
105 
106 		/* write 8-byte SOP chunk data */
107 		while (dest < send) {
108 			writeq(*(u64 *)from, dest);
109 			from += sizeof(u64);
110 			dest += sizeof(u64);
111 		}
112 		/* drop out of the SOP range */
113 		dest -= SOP_DISTANCE;
114 		dend -= SOP_DISTANCE;
115 
116 		/*
117 		 * If the wrap comes before or matches the data end,
118 		 * copy until until the wrap, then wrap.
119 		 *
120 		 * If the data ends at the end of the SOP above and
121 		 * the buffer wraps, then pbuf->end == dend == dest
122 		 * and nothing will get written, but we will wrap in
123 		 * case there is a dangling DWORD.
124 		 */
125 		if (pbuf->end <= dend) {
126 			while (dest < pbuf->end) {
127 				writeq(*(u64 *)from, dest);
128 				from += sizeof(u64);
129 				dest += sizeof(u64);
130 			}
131 
132 			dest -= pbuf->sc->size;
133 			dend -= pbuf->sc->size;
134 		}
135 
136 		/* write 8-byte non-SOP, non-wrap chunk data */
137 		while (dest < dend) {
138 			writeq(*(u64 *)from, dest);
139 			from += sizeof(u64);
140 			dest += sizeof(u64);
141 		}
142 	}
143 	/* at this point we have wrapped if we are going to wrap */
144 
145 	/* write dangling u32, if any */
146 	if (count & 1) {
147 		union mix val;
148 
149 		val.val64 = 0;
150 		val.val32[0] = *(u32 *)from;
151 		writeq(val.val64, dest);
152 		dest += sizeof(u64);
153 	}
154 	/*
155 	 * fill in rest of block, no need to check pbuf->end
156 	 * as we only wrap on a block boundary
157 	 */
158 	while (((unsigned long)dest & PIO_BLOCK_MASK) != 0) {
159 		writeq(0, dest);
160 		dest += sizeof(u64);
161 	}
162 
163 	/* finished with this buffer */
164 	this_cpu_dec(*pbuf->sc->buffers_allocated);
165 	preempt_enable();
166 }
167 
168 /*
169  * Handle carry bytes using shifts and masks.
170  *
171  * NOTE: the value the unused portion of carry is expected to always be zero.
172  */
173 
174 /*
175  * "zero" shift - bit shift used to zero out upper bytes.  Input is
176  * the count of LSB bytes to preserve.
177  */
178 #define zshift(x) (8 * (8 - (x)))
179 
180 /*
181  * "merge" shift - bit shift used to merge with carry bytes.  Input is
182  * the LSB byte count to move beyond.
183  */
184 #define mshift(x) (8 * (x))
185 
186 /*
187  * Jump copy - no-loop copy for < 8 bytes.
188  */
189 static inline void jcopy(u8 *dest, const u8 *src, u32 n)
190 {
191 	switch (n) {
192 	case 7:
193 		*dest++ = *src++;
194 		/* fall through */
195 	case 6:
196 		*dest++ = *src++;
197 		/* fall through */
198 	case 5:
199 		*dest++ = *src++;
200 		/* fall through */
201 	case 4:
202 		*dest++ = *src++;
203 		/* fall through */
204 	case 3:
205 		*dest++ = *src++;
206 		/* fall through */
207 	case 2:
208 		*dest++ = *src++;
209 		/* fall through */
210 	case 1:
211 		*dest++ = *src++;
212 		/* fall through */
213 	}
214 }
215 
216 /*
217  * Read nbytes from "from" and and place them in the low bytes
218  * of pbuf->carry.  Other bytes are left as-is.  Any previous
219  * value in pbuf->carry is lost.
220  *
221  * NOTES:
222  * o do not read from from if nbytes is zero
223  * o from may _not_ be u64 aligned.
224  */
225 static inline void read_low_bytes(struct pio_buf *pbuf, const void *from,
226 				  unsigned int nbytes)
227 {
228 	pbuf->carry.val64 = 0;
229 	jcopy(&pbuf->carry.val8[0], from, nbytes);
230 	pbuf->carry_bytes = nbytes;
231 }
232 
233 /*
234  * Read nbytes bytes from "from" and put them at the end of pbuf->carry.
235  * It is expected that the extra read does not overfill carry.
236  *
237  * NOTES:
238  * o from may _not_ be u64 aligned
239  * o nbytes may span a QW boundary
240  */
241 static inline void read_extra_bytes(struct pio_buf *pbuf,
242 				    const void *from, unsigned int nbytes)
243 {
244 	jcopy(&pbuf->carry.val8[pbuf->carry_bytes], from, nbytes);
245 	pbuf->carry_bytes += nbytes;
246 }
247 
248 /*
249  * Write a quad word using parts of pbuf->carry and the next 8 bytes of src.
250  * Put the unused part of the next 8 bytes of src into the LSB bytes of
251  * pbuf->carry with the upper bytes zeroed..
252  *
253  * NOTES:
254  * o result must keep unused bytes zeroed
255  * o src must be u64 aligned
256  */
257 static inline void merge_write8(
258 	struct pio_buf *pbuf,
259 	void __iomem *dest,
260 	const void *src)
261 {
262 	u64 new, temp;
263 
264 	new = *(u64 *)src;
265 	temp = pbuf->carry.val64 | (new << mshift(pbuf->carry_bytes));
266 	writeq(temp, dest);
267 	pbuf->carry.val64 = new >> zshift(pbuf->carry_bytes);
268 }
269 
270 /*
271  * Write a quad word using all bytes of carry.
272  */
273 static inline void carry8_write8(union mix carry, void __iomem *dest)
274 {
275 	writeq(carry.val64, dest);
276 }
277 
278 /*
279  * Write a quad word using all the valid bytes of carry.  If carry
280  * has zero valid bytes, nothing is written.
281  * Returns 0 on nothing written, non-zero on quad word written.
282  */
283 static inline int carry_write8(struct pio_buf *pbuf, void __iomem *dest)
284 {
285 	if (pbuf->carry_bytes) {
286 		/* unused bytes are always kept zeroed, so just write */
287 		writeq(pbuf->carry.val64, dest);
288 		return 1;
289 	}
290 
291 	return 0;
292 }
293 
294 /*
295  * Segmented PIO Copy - start
296  *
297  * Start a PIO copy.
298  *
299  * @pbuf: destination buffer
300  * @pbc: the PBC for the PIO buffer
301  * @from: data source, QWORD aligned
302  * @nbytes: bytes to copy
303  */
304 void seg_pio_copy_start(struct pio_buf *pbuf, u64 pbc,
305 			const void *from, size_t nbytes)
306 {
307 	void __iomem *dest = pbuf->start + SOP_DISTANCE;
308 	void __iomem *send = dest + PIO_BLOCK_SIZE;
309 	void __iomem *dend;			/* 8-byte data end */
310 
311 	writeq(pbc, dest);
312 	dest += sizeof(u64);
313 
314 	/* calculate where the QWORD data ends - in SOP=1 space */
315 	dend = dest + ((nbytes >> 3) * sizeof(u64));
316 
317 	if (dend < send) {
318 		/*
319 		 * all QWORD data is within the SOP block, does *not*
320 		 * reach the end of the SOP block
321 		 */
322 
323 		while (dest < dend) {
324 			writeq(*(u64 *)from, dest);
325 			from += sizeof(u64);
326 			dest += sizeof(u64);
327 		}
328 		/*
329 		 * No boundary checks are needed here:
330 		 * 0. We're not on the SOP block boundary
331 		 * 1. The possible DWORD dangle will still be within
332 		 *    the SOP block
333 		 * 2. We cannot wrap except on a block boundary.
334 		 */
335 	} else {
336 		/* QWORD data extends _to_ or beyond the SOP block */
337 
338 		/* write 8-byte SOP chunk data */
339 		while (dest < send) {
340 			writeq(*(u64 *)from, dest);
341 			from += sizeof(u64);
342 			dest += sizeof(u64);
343 		}
344 		/* drop out of the SOP range */
345 		dest -= SOP_DISTANCE;
346 		dend -= SOP_DISTANCE;
347 
348 		/*
349 		 * If the wrap comes before or matches the data end,
350 		 * copy until until the wrap, then wrap.
351 		 *
352 		 * If the data ends at the end of the SOP above and
353 		 * the buffer wraps, then pbuf->end == dend == dest
354 		 * and nothing will get written, but we will wrap in
355 		 * case there is a dangling DWORD.
356 		 */
357 		if (pbuf->end <= dend) {
358 			while (dest < pbuf->end) {
359 				writeq(*(u64 *)from, dest);
360 				from += sizeof(u64);
361 				dest += sizeof(u64);
362 			}
363 
364 			dest -= pbuf->sc->size;
365 			dend -= pbuf->sc->size;
366 		}
367 
368 		/* write 8-byte non-SOP, non-wrap chunk data */
369 		while (dest < dend) {
370 			writeq(*(u64 *)from, dest);
371 			from += sizeof(u64);
372 			dest += sizeof(u64);
373 		}
374 	}
375 	/* at this point we have wrapped if we are going to wrap */
376 
377 	/* ...but it doesn't matter as we're done writing */
378 
379 	/* save dangling bytes, if any */
380 	read_low_bytes(pbuf, from, nbytes & 0x7);
381 
382 	pbuf->qw_written = 1 /*PBC*/ + (nbytes >> 3);
383 }
384 
385 /*
386  * Mid copy helper, "mixed case" - source is 64-bit aligned but carry
387  * bytes are non-zero.
388  *
389  * Whole u64s must be written to the chip, so bytes must be manually merged.
390  *
391  * @pbuf: destination buffer
392  * @from: data source, is QWORD aligned.
393  * @nbytes: bytes to copy
394  *
395  * Must handle nbytes < 8.
396  */
397 static void mid_copy_mix(struct pio_buf *pbuf, const void *from, size_t nbytes)
398 {
399 	void __iomem *dest = pbuf->start + (pbuf->qw_written * sizeof(u64));
400 	void __iomem *dend;			/* 8-byte data end */
401 	unsigned long qw_to_write = nbytes >> 3;
402 	unsigned long bytes_left = nbytes & 0x7;
403 
404 	/* calculate 8-byte data end */
405 	dend = dest + (qw_to_write * sizeof(u64));
406 
407 	if (pbuf->qw_written < PIO_BLOCK_QWS) {
408 		/*
409 		 * Still within SOP block.  We don't need to check for
410 		 * wrap because we are still in the first block and
411 		 * can only wrap on block boundaries.
412 		 */
413 		void __iomem *send;		/* SOP end */
414 		void __iomem *xend;
415 
416 		/*
417 		 * calculate the end of data or end of block, whichever
418 		 * comes first
419 		 */
420 		send = pbuf->start + PIO_BLOCK_SIZE;
421 		xend = min(send, dend);
422 
423 		/* shift up to SOP=1 space */
424 		dest += SOP_DISTANCE;
425 		xend += SOP_DISTANCE;
426 
427 		/* write 8-byte chunk data */
428 		while (dest < xend) {
429 			merge_write8(pbuf, dest, from);
430 			from += sizeof(u64);
431 			dest += sizeof(u64);
432 		}
433 
434 		/* shift down to SOP=0 space */
435 		dest -= SOP_DISTANCE;
436 	}
437 	/*
438 	 * At this point dest could be (either, both, or neither):
439 	 * - at dend
440 	 * - at the wrap
441 	 */
442 
443 	/*
444 	 * If the wrap comes before or matches the data end,
445 	 * copy until until the wrap, then wrap.
446 	 *
447 	 * If dest is at the wrap, we will fall into the if,
448 	 * not do the loop, when wrap.
449 	 *
450 	 * If the data ends at the end of the SOP above and
451 	 * the buffer wraps, then pbuf->end == dend == dest
452 	 * and nothing will get written.
453 	 */
454 	if (pbuf->end <= dend) {
455 		while (dest < pbuf->end) {
456 			merge_write8(pbuf, dest, from);
457 			from += sizeof(u64);
458 			dest += sizeof(u64);
459 		}
460 
461 		dest -= pbuf->sc->size;
462 		dend -= pbuf->sc->size;
463 	}
464 
465 	/* write 8-byte non-SOP, non-wrap chunk data */
466 	while (dest < dend) {
467 		merge_write8(pbuf, dest, from);
468 		from += sizeof(u64);
469 		dest += sizeof(u64);
470 	}
471 
472 	pbuf->qw_written += qw_to_write;
473 
474 	/* handle carry and left-over bytes */
475 	if (pbuf->carry_bytes + bytes_left >= 8) {
476 		unsigned long nread;
477 
478 		/* there is enough to fill another qw - fill carry */
479 		nread = 8 - pbuf->carry_bytes;
480 		read_extra_bytes(pbuf, from, nread);
481 
482 		/*
483 		 * One more write - but need to make sure dest is correct.
484 		 * Check for wrap and the possibility the write
485 		 * should be in SOP space.
486 		 *
487 		 * The two checks immediately below cannot both be true, hence
488 		 * the else. If we have wrapped, we cannot still be within the
489 		 * first block. Conversely, if we are still in the first block,
490 		 * we cannot have wrapped. We do the wrap check first as that
491 		 * is more likely.
492 		 */
493 		/* adjust if we have wrapped */
494 		if (dest >= pbuf->end)
495 			dest -= pbuf->sc->size;
496 		/* jump to the SOP range if within the first block */
497 		else if (pbuf->qw_written < PIO_BLOCK_QWS)
498 			dest += SOP_DISTANCE;
499 
500 		/* flush out full carry */
501 		carry8_write8(pbuf->carry, dest);
502 		pbuf->qw_written++;
503 
504 		/* now adjust and read the rest of the bytes into carry */
505 		bytes_left -= nread;
506 		from += nread; /* from is now not aligned */
507 		read_low_bytes(pbuf, from, bytes_left);
508 	} else {
509 		/* not enough to fill another qw, append the rest to carry */
510 		read_extra_bytes(pbuf, from, bytes_left);
511 	}
512 }
513 
514 /*
515  * Mid copy helper, "straight case" - source pointer is 64-bit aligned
516  * with no carry bytes.
517  *
518  * @pbuf: destination buffer
519  * @from: data source, is QWORD aligned
520  * @nbytes: bytes to copy
521  *
522  * Must handle nbytes < 8.
523  */
524 static void mid_copy_straight(struct pio_buf *pbuf,
525 			      const void *from, size_t nbytes)
526 {
527 	void __iomem *dest = pbuf->start + (pbuf->qw_written * sizeof(u64));
528 	void __iomem *dend;			/* 8-byte data end */
529 
530 	/* calculate 8-byte data end */
531 	dend = dest + ((nbytes >> 3) * sizeof(u64));
532 
533 	if (pbuf->qw_written < PIO_BLOCK_QWS) {
534 		/*
535 		 * Still within SOP block.  We don't need to check for
536 		 * wrap because we are still in the first block and
537 		 * can only wrap on block boundaries.
538 		 */
539 		void __iomem *send;		/* SOP end */
540 		void __iomem *xend;
541 
542 		/*
543 		 * calculate the end of data or end of block, whichever
544 		 * comes first
545 		 */
546 		send = pbuf->start + PIO_BLOCK_SIZE;
547 		xend = min(send, dend);
548 
549 		/* shift up to SOP=1 space */
550 		dest += SOP_DISTANCE;
551 		xend += SOP_DISTANCE;
552 
553 		/* write 8-byte chunk data */
554 		while (dest < xend) {
555 			writeq(*(u64 *)from, dest);
556 			from += sizeof(u64);
557 			dest += sizeof(u64);
558 		}
559 
560 		/* shift down to SOP=0 space */
561 		dest -= SOP_DISTANCE;
562 	}
563 	/*
564 	 * At this point dest could be (either, both, or neither):
565 	 * - at dend
566 	 * - at the wrap
567 	 */
568 
569 	/*
570 	 * If the wrap comes before or matches the data end,
571 	 * copy until until the wrap, then wrap.
572 	 *
573 	 * If dest is at the wrap, we will fall into the if,
574 	 * not do the loop, when wrap.
575 	 *
576 	 * If the data ends at the end of the SOP above and
577 	 * the buffer wraps, then pbuf->end == dend == dest
578 	 * and nothing will get written.
579 	 */
580 	if (pbuf->end <= dend) {
581 		while (dest < pbuf->end) {
582 			writeq(*(u64 *)from, dest);
583 			from += sizeof(u64);
584 			dest += sizeof(u64);
585 		}
586 
587 		dest -= pbuf->sc->size;
588 		dend -= pbuf->sc->size;
589 	}
590 
591 	/* write 8-byte non-SOP, non-wrap chunk data */
592 	while (dest < dend) {
593 		writeq(*(u64 *)from, dest);
594 		from += sizeof(u64);
595 		dest += sizeof(u64);
596 	}
597 
598 	/* we know carry_bytes was zero on entry to this routine */
599 	read_low_bytes(pbuf, from, nbytes & 0x7);
600 
601 	pbuf->qw_written += nbytes >> 3;
602 }
603 
604 /*
605  * Segmented PIO Copy - middle
606  *
607  * Must handle any aligned tail and any aligned source with any byte count.
608  *
609  * @pbuf: a number of blocks allocated within a PIO send context
610  * @from: data source
611  * @nbytes: number of bytes to copy
612  */
613 void seg_pio_copy_mid(struct pio_buf *pbuf, const void *from, size_t nbytes)
614 {
615 	unsigned long from_align = (unsigned long)from & 0x7;
616 
617 	if (pbuf->carry_bytes + nbytes < 8) {
618 		/* not enough bytes to fill a QW */
619 		read_extra_bytes(pbuf, from, nbytes);
620 		return;
621 	}
622 
623 	if (from_align) {
624 		/* misaligned source pointer - align it */
625 		unsigned long to_align;
626 
627 		/* bytes to read to align "from" */
628 		to_align = 8 - from_align;
629 
630 		/*
631 		 * In the advance-to-alignment logic below, we do not need
632 		 * to check if we are using more than nbytes.  This is because
633 		 * if we are here, we already know that carry+nbytes will
634 		 * fill at least one QW.
635 		 */
636 		if (pbuf->carry_bytes + to_align < 8) {
637 			/* not enough align bytes to fill a QW */
638 			read_extra_bytes(pbuf, from, to_align);
639 			from += to_align;
640 			nbytes -= to_align;
641 		} else {
642 			/* bytes to fill carry */
643 			unsigned long to_fill = 8 - pbuf->carry_bytes;
644 			/* bytes left over to be read */
645 			unsigned long extra = to_align - to_fill;
646 			void __iomem *dest;
647 
648 			/* fill carry... */
649 			read_extra_bytes(pbuf, from, to_fill);
650 			from += to_fill;
651 			nbytes -= to_fill;
652 			/* may not be enough valid bytes left to align */
653 			if (extra > nbytes)
654 				extra = nbytes;
655 
656 			/* ...now write carry */
657 			dest = pbuf->start + (pbuf->qw_written * sizeof(u64));
658 
659 			/*
660 			 * The two checks immediately below cannot both be
661 			 * true, hence the else.  If we have wrapped, we
662 			 * cannot still be within the first block.
663 			 * Conversely, if we are still in the first block, we
664 			 * cannot have wrapped.  We do the wrap check first
665 			 * as that is more likely.
666 			 */
667 			/* adjust if we've wrapped */
668 			if (dest >= pbuf->end)
669 				dest -= pbuf->sc->size;
670 			/* jump to SOP range if within the first block */
671 			else if (pbuf->qw_written < PIO_BLOCK_QWS)
672 				dest += SOP_DISTANCE;
673 
674 			carry8_write8(pbuf->carry, dest);
675 			pbuf->qw_written++;
676 
677 			/* read any extra bytes to do final alignment */
678 			/* this will overwrite anything in pbuf->carry */
679 			read_low_bytes(pbuf, from, extra);
680 			from += extra;
681 			nbytes -= extra;
682 			/*
683 			 * If no bytes are left, return early - we are done.
684 			 * NOTE: This short-circuit is *required* because
685 			 * "extra" may have been reduced in size and "from"
686 			 * is not aligned, as required when leaving this
687 			 * if block.
688 			 */
689 			if (nbytes == 0)
690 				return;
691 		}
692 
693 		/* at this point, from is QW aligned */
694 	}
695 
696 	if (pbuf->carry_bytes)
697 		mid_copy_mix(pbuf, from, nbytes);
698 	else
699 		mid_copy_straight(pbuf, from, nbytes);
700 }
701 
702 /*
703  * Segmented PIO Copy - end
704  *
705  * Write any remainder (in pbuf->carry) and finish writing the whole block.
706  *
707  * @pbuf: a number of blocks allocated within a PIO send context
708  */
709 void seg_pio_copy_end(struct pio_buf *pbuf)
710 {
711 	void __iomem *dest = pbuf->start + (pbuf->qw_written * sizeof(u64));
712 
713 	/*
714 	 * The two checks immediately below cannot both be true, hence the
715 	 * else.  If we have wrapped, we cannot still be within the first
716 	 * block.  Conversely, if we are still in the first block, we
717 	 * cannot have wrapped.  We do the wrap check first as that is
718 	 * more likely.
719 	 */
720 	/* adjust if we have wrapped */
721 	if (dest >= pbuf->end)
722 		dest -= pbuf->sc->size;
723 	/* jump to the SOP range if within the first block */
724 	else if (pbuf->qw_written < PIO_BLOCK_QWS)
725 		dest += SOP_DISTANCE;
726 
727 	/* write final bytes, if any */
728 	if (carry_write8(pbuf, dest)) {
729 		dest += sizeof(u64);
730 		/*
731 		 * NOTE: We do not need to recalculate whether dest needs
732 		 * SOP_DISTANCE or not.
733 		 *
734 		 * If we are in the first block and the dangle write
735 		 * keeps us in the same block, dest will need
736 		 * to retain SOP_DISTANCE in the loop below.
737 		 *
738 		 * If we are in the first block and the dangle write pushes
739 		 * us to the next block, then loop below will not run
740 		 * and dest is not used.  Hence we do not need to update
741 		 * it.
742 		 *
743 		 * If we are past the first block, then SOP_DISTANCE
744 		 * was never added, so there is nothing to do.
745 		 */
746 	}
747 
748 	/* fill in rest of block */
749 	while (((unsigned long)dest & PIO_BLOCK_MASK) != 0) {
750 		writeq(0, dest);
751 		dest += sizeof(u64);
752 	}
753 
754 	/* finished with this buffer */
755 	this_cpu_dec(*pbuf->sc->buffers_allocated);
756 	preempt_enable();
757 }
758