1 // SPDX-License-Identifier: GPL-2.0 or BSD-3-Clause
2 /*
3  * Copyright(c) 2015, 2016 Intel Corporation.
4  */
5 
6 #include "hfi.h"
7 
8 /* additive distance between non-SOP and SOP space */
9 #define SOP_DISTANCE (TXE_PIO_SIZE / 2)
10 #define PIO_BLOCK_MASK (PIO_BLOCK_SIZE - 1)
11 /* number of QUADWORDs in a block */
12 #define PIO_BLOCK_QWS (PIO_BLOCK_SIZE / sizeof(u64))
13 
14 /**
15  * pio_copy - copy data block to MMIO space
16  * @dd: hfi1 dev data
17  * @pbuf: a number of blocks allocated within a PIO send context
18  * @pbc: PBC to send
19  * @from: source, must be 8 byte aligned
20  * @count: number of DWORD (32-bit) quantities to copy from source
21  *
22  * Copy data from source to PIO Send Buffer memory, 8 bytes at a time.
23  * Must always write full BLOCK_SIZE bytes blocks.  The first block must
24  * be written to the corresponding SOP=1 address.
25  *
26  * Known:
27  * o pbuf->start always starts on a block boundary
28  * o pbuf can wrap only at a block boundary
29  */
pio_copy(struct hfi1_devdata * dd,struct pio_buf * pbuf,u64 pbc,const void * from,size_t count)30 void pio_copy(struct hfi1_devdata *dd, struct pio_buf *pbuf, u64 pbc,
31 	      const void *from, size_t count)
32 {
33 	void __iomem *dest = pbuf->start + SOP_DISTANCE;
34 	void __iomem *send = dest + PIO_BLOCK_SIZE;
35 	void __iomem *dend;			/* 8-byte data end */
36 
37 	/* write the PBC */
38 	writeq(pbc, dest);
39 	dest += sizeof(u64);
40 
41 	/* calculate where the QWORD data ends - in SOP=1 space */
42 	dend = dest + ((count >> 1) * sizeof(u64));
43 
44 	if (dend < send) {
45 		/*
46 		 * all QWORD data is within the SOP block, does *not*
47 		 * reach the end of the SOP block
48 		 */
49 
50 		while (dest < dend) {
51 			writeq(*(u64 *)from, dest);
52 			from += sizeof(u64);
53 			dest += sizeof(u64);
54 		}
55 		/*
56 		 * No boundary checks are needed here:
57 		 * 0. We're not on the SOP block boundary
58 		 * 1. The possible DWORD dangle will still be within
59 		 *    the SOP block
60 		 * 2. We cannot wrap except on a block boundary.
61 		 */
62 	} else {
63 		/* QWORD data extends _to_ or beyond the SOP block */
64 
65 		/* write 8-byte SOP chunk data */
66 		while (dest < send) {
67 			writeq(*(u64 *)from, dest);
68 			from += sizeof(u64);
69 			dest += sizeof(u64);
70 		}
71 		/* drop out of the SOP range */
72 		dest -= SOP_DISTANCE;
73 		dend -= SOP_DISTANCE;
74 
75 		/*
76 		 * If the wrap comes before or matches the data end,
77 		 * copy until until the wrap, then wrap.
78 		 *
79 		 * If the data ends at the end of the SOP above and
80 		 * the buffer wraps, then pbuf->end == dend == dest
81 		 * and nothing will get written, but we will wrap in
82 		 * case there is a dangling DWORD.
83 		 */
84 		if (pbuf->end <= dend) {
85 			while (dest < pbuf->end) {
86 				writeq(*(u64 *)from, dest);
87 				from += sizeof(u64);
88 				dest += sizeof(u64);
89 			}
90 
91 			dest -= pbuf->sc->size;
92 			dend -= pbuf->sc->size;
93 		}
94 
95 		/* write 8-byte non-SOP, non-wrap chunk data */
96 		while (dest < dend) {
97 			writeq(*(u64 *)from, dest);
98 			from += sizeof(u64);
99 			dest += sizeof(u64);
100 		}
101 	}
102 	/* at this point we have wrapped if we are going to wrap */
103 
104 	/* write dangling u32, if any */
105 	if (count & 1) {
106 		union mix val;
107 
108 		val.val64 = 0;
109 		val.val32[0] = *(u32 *)from;
110 		writeq(val.val64, dest);
111 		dest += sizeof(u64);
112 	}
113 	/*
114 	 * fill in rest of block, no need to check pbuf->end
115 	 * as we only wrap on a block boundary
116 	 */
117 	while (((unsigned long)dest & PIO_BLOCK_MASK) != 0) {
118 		writeq(0, dest);
119 		dest += sizeof(u64);
120 	}
121 
122 	/* finished with this buffer */
123 	this_cpu_dec(*pbuf->sc->buffers_allocated);
124 	preempt_enable();
125 }
126 
127 /*
128  * Handle carry bytes using shifts and masks.
129  *
130  * NOTE: the value the unused portion of carry is expected to always be zero.
131  */
132 
133 /*
134  * "zero" shift - bit shift used to zero out upper bytes.  Input is
135  * the count of LSB bytes to preserve.
136  */
137 #define zshift(x) (8 * (8 - (x)))
138 
139 /*
140  * "merge" shift - bit shift used to merge with carry bytes.  Input is
141  * the LSB byte count to move beyond.
142  */
143 #define mshift(x) (8 * (x))
144 
145 /*
146  * Jump copy - no-loop copy for < 8 bytes.
147  */
jcopy(u8 * dest,const u8 * src,u32 n)148 static inline void jcopy(u8 *dest, const u8 *src, u32 n)
149 {
150 	switch (n) {
151 	case 7:
152 		*dest++ = *src++;
153 		fallthrough;
154 	case 6:
155 		*dest++ = *src++;
156 		fallthrough;
157 	case 5:
158 		*dest++ = *src++;
159 		fallthrough;
160 	case 4:
161 		*dest++ = *src++;
162 		fallthrough;
163 	case 3:
164 		*dest++ = *src++;
165 		fallthrough;
166 	case 2:
167 		*dest++ = *src++;
168 		fallthrough;
169 	case 1:
170 		*dest++ = *src++;
171 	}
172 }
173 
174 /*
175  * Read nbytes from "from" and place them in the low bytes
176  * of pbuf->carry.  Other bytes are left as-is.  Any previous
177  * value in pbuf->carry is lost.
178  *
179  * NOTES:
180  * o do not read from from if nbytes is zero
181  * o from may _not_ be u64 aligned.
182  */
read_low_bytes(struct pio_buf * pbuf,const void * from,unsigned int nbytes)183 static inline void read_low_bytes(struct pio_buf *pbuf, const void *from,
184 				  unsigned int nbytes)
185 {
186 	pbuf->carry.val64 = 0;
187 	jcopy(&pbuf->carry.val8[0], from, nbytes);
188 	pbuf->carry_bytes = nbytes;
189 }
190 
191 /*
192  * Read nbytes bytes from "from" and put them at the end of pbuf->carry.
193  * It is expected that the extra read does not overfill carry.
194  *
195  * NOTES:
196  * o from may _not_ be u64 aligned
197  * o nbytes may span a QW boundary
198  */
read_extra_bytes(struct pio_buf * pbuf,const void * from,unsigned int nbytes)199 static inline void read_extra_bytes(struct pio_buf *pbuf,
200 				    const void *from, unsigned int nbytes)
201 {
202 	jcopy(&pbuf->carry.val8[pbuf->carry_bytes], from, nbytes);
203 	pbuf->carry_bytes += nbytes;
204 }
205 
206 /*
207  * Write a quad word using parts of pbuf->carry and the next 8 bytes of src.
208  * Put the unused part of the next 8 bytes of src into the LSB bytes of
209  * pbuf->carry with the upper bytes zeroed..
210  *
211  * NOTES:
212  * o result must keep unused bytes zeroed
213  * o src must be u64 aligned
214  */
merge_write8(struct pio_buf * pbuf,void __iomem * dest,const void * src)215 static inline void merge_write8(
216 	struct pio_buf *pbuf,
217 	void __iomem *dest,
218 	const void *src)
219 {
220 	u64 new, temp;
221 
222 	new = *(u64 *)src;
223 	temp = pbuf->carry.val64 | (new << mshift(pbuf->carry_bytes));
224 	writeq(temp, dest);
225 	pbuf->carry.val64 = new >> zshift(pbuf->carry_bytes);
226 }
227 
228 /*
229  * Write a quad word using all bytes of carry.
230  */
carry8_write8(union mix carry,void __iomem * dest)231 static inline void carry8_write8(union mix carry, void __iomem *dest)
232 {
233 	writeq(carry.val64, dest);
234 }
235 
236 /*
237  * Write a quad word using all the valid bytes of carry.  If carry
238  * has zero valid bytes, nothing is written.
239  * Returns 0 on nothing written, non-zero on quad word written.
240  */
carry_write8(struct pio_buf * pbuf,void __iomem * dest)241 static inline int carry_write8(struct pio_buf *pbuf, void __iomem *dest)
242 {
243 	if (pbuf->carry_bytes) {
244 		/* unused bytes are always kept zeroed, so just write */
245 		writeq(pbuf->carry.val64, dest);
246 		return 1;
247 	}
248 
249 	return 0;
250 }
251 
252 /*
253  * Segmented PIO Copy - start
254  *
255  * Start a PIO copy.
256  *
257  * @pbuf: destination buffer
258  * @pbc: the PBC for the PIO buffer
259  * @from: data source, QWORD aligned
260  * @nbytes: bytes to copy
261  */
seg_pio_copy_start(struct pio_buf * pbuf,u64 pbc,const void * from,size_t nbytes)262 void seg_pio_copy_start(struct pio_buf *pbuf, u64 pbc,
263 			const void *from, size_t nbytes)
264 {
265 	void __iomem *dest = pbuf->start + SOP_DISTANCE;
266 	void __iomem *send = dest + PIO_BLOCK_SIZE;
267 	void __iomem *dend;			/* 8-byte data end */
268 
269 	writeq(pbc, dest);
270 	dest += sizeof(u64);
271 
272 	/* calculate where the QWORD data ends - in SOP=1 space */
273 	dend = dest + ((nbytes >> 3) * sizeof(u64));
274 
275 	if (dend < send) {
276 		/*
277 		 * all QWORD data is within the SOP block, does *not*
278 		 * reach the end of the SOP block
279 		 */
280 
281 		while (dest < dend) {
282 			writeq(*(u64 *)from, dest);
283 			from += sizeof(u64);
284 			dest += sizeof(u64);
285 		}
286 		/*
287 		 * No boundary checks are needed here:
288 		 * 0. We're not on the SOP block boundary
289 		 * 1. The possible DWORD dangle will still be within
290 		 *    the SOP block
291 		 * 2. We cannot wrap except on a block boundary.
292 		 */
293 	} else {
294 		/* QWORD data extends _to_ or beyond the SOP block */
295 
296 		/* write 8-byte SOP chunk data */
297 		while (dest < send) {
298 			writeq(*(u64 *)from, dest);
299 			from += sizeof(u64);
300 			dest += sizeof(u64);
301 		}
302 		/* drop out of the SOP range */
303 		dest -= SOP_DISTANCE;
304 		dend -= SOP_DISTANCE;
305 
306 		/*
307 		 * If the wrap comes before or matches the data end,
308 		 * copy until until the wrap, then wrap.
309 		 *
310 		 * If the data ends at the end of the SOP above and
311 		 * the buffer wraps, then pbuf->end == dend == dest
312 		 * and nothing will get written, but we will wrap in
313 		 * case there is a dangling DWORD.
314 		 */
315 		if (pbuf->end <= dend) {
316 			while (dest < pbuf->end) {
317 				writeq(*(u64 *)from, dest);
318 				from += sizeof(u64);
319 				dest += sizeof(u64);
320 			}
321 
322 			dest -= pbuf->sc->size;
323 			dend -= pbuf->sc->size;
324 		}
325 
326 		/* write 8-byte non-SOP, non-wrap chunk data */
327 		while (dest < dend) {
328 			writeq(*(u64 *)from, dest);
329 			from += sizeof(u64);
330 			dest += sizeof(u64);
331 		}
332 	}
333 	/* at this point we have wrapped if we are going to wrap */
334 
335 	/* ...but it doesn't matter as we're done writing */
336 
337 	/* save dangling bytes, if any */
338 	read_low_bytes(pbuf, from, nbytes & 0x7);
339 
340 	pbuf->qw_written = 1 /*PBC*/ + (nbytes >> 3);
341 }
342 
343 /*
344  * Mid copy helper, "mixed case" - source is 64-bit aligned but carry
345  * bytes are non-zero.
346  *
347  * Whole u64s must be written to the chip, so bytes must be manually merged.
348  *
349  * @pbuf: destination buffer
350  * @from: data source, is QWORD aligned.
351  * @nbytes: bytes to copy
352  *
353  * Must handle nbytes < 8.
354  */
mid_copy_mix(struct pio_buf * pbuf,const void * from,size_t nbytes)355 static void mid_copy_mix(struct pio_buf *pbuf, const void *from, size_t nbytes)
356 {
357 	void __iomem *dest = pbuf->start + (pbuf->qw_written * sizeof(u64));
358 	void __iomem *dend;			/* 8-byte data end */
359 	unsigned long qw_to_write = nbytes >> 3;
360 	unsigned long bytes_left = nbytes & 0x7;
361 
362 	/* calculate 8-byte data end */
363 	dend = dest + (qw_to_write * sizeof(u64));
364 
365 	if (pbuf->qw_written < PIO_BLOCK_QWS) {
366 		/*
367 		 * Still within SOP block.  We don't need to check for
368 		 * wrap because we are still in the first block and
369 		 * can only wrap on block boundaries.
370 		 */
371 		void __iomem *send;		/* SOP end */
372 		void __iomem *xend;
373 
374 		/*
375 		 * calculate the end of data or end of block, whichever
376 		 * comes first
377 		 */
378 		send = pbuf->start + PIO_BLOCK_SIZE;
379 		xend = min(send, dend);
380 
381 		/* shift up to SOP=1 space */
382 		dest += SOP_DISTANCE;
383 		xend += SOP_DISTANCE;
384 
385 		/* write 8-byte chunk data */
386 		while (dest < xend) {
387 			merge_write8(pbuf, dest, from);
388 			from += sizeof(u64);
389 			dest += sizeof(u64);
390 		}
391 
392 		/* shift down to SOP=0 space */
393 		dest -= SOP_DISTANCE;
394 	}
395 	/*
396 	 * At this point dest could be (either, both, or neither):
397 	 * - at dend
398 	 * - at the wrap
399 	 */
400 
401 	/*
402 	 * If the wrap comes before or matches the data end,
403 	 * copy until until the wrap, then wrap.
404 	 *
405 	 * If dest is at the wrap, we will fall into the if,
406 	 * not do the loop, when wrap.
407 	 *
408 	 * If the data ends at the end of the SOP above and
409 	 * the buffer wraps, then pbuf->end == dend == dest
410 	 * and nothing will get written.
411 	 */
412 	if (pbuf->end <= dend) {
413 		while (dest < pbuf->end) {
414 			merge_write8(pbuf, dest, from);
415 			from += sizeof(u64);
416 			dest += sizeof(u64);
417 		}
418 
419 		dest -= pbuf->sc->size;
420 		dend -= pbuf->sc->size;
421 	}
422 
423 	/* write 8-byte non-SOP, non-wrap chunk data */
424 	while (dest < dend) {
425 		merge_write8(pbuf, dest, from);
426 		from += sizeof(u64);
427 		dest += sizeof(u64);
428 	}
429 
430 	pbuf->qw_written += qw_to_write;
431 
432 	/* handle carry and left-over bytes */
433 	if (pbuf->carry_bytes + bytes_left >= 8) {
434 		unsigned long nread;
435 
436 		/* there is enough to fill another qw - fill carry */
437 		nread = 8 - pbuf->carry_bytes;
438 		read_extra_bytes(pbuf, from, nread);
439 
440 		/*
441 		 * One more write - but need to make sure dest is correct.
442 		 * Check for wrap and the possibility the write
443 		 * should be in SOP space.
444 		 *
445 		 * The two checks immediately below cannot both be true, hence
446 		 * the else. If we have wrapped, we cannot still be within the
447 		 * first block. Conversely, if we are still in the first block,
448 		 * we cannot have wrapped. We do the wrap check first as that
449 		 * is more likely.
450 		 */
451 		/* adjust if we have wrapped */
452 		if (dest >= pbuf->end)
453 			dest -= pbuf->sc->size;
454 		/* jump to the SOP range if within the first block */
455 		else if (pbuf->qw_written < PIO_BLOCK_QWS)
456 			dest += SOP_DISTANCE;
457 
458 		/* flush out full carry */
459 		carry8_write8(pbuf->carry, dest);
460 		pbuf->qw_written++;
461 
462 		/* now adjust and read the rest of the bytes into carry */
463 		bytes_left -= nread;
464 		from += nread; /* from is now not aligned */
465 		read_low_bytes(pbuf, from, bytes_left);
466 	} else {
467 		/* not enough to fill another qw, append the rest to carry */
468 		read_extra_bytes(pbuf, from, bytes_left);
469 	}
470 }
471 
472 /*
473  * Mid copy helper, "straight case" - source pointer is 64-bit aligned
474  * with no carry bytes.
475  *
476  * @pbuf: destination buffer
477  * @from: data source, is QWORD aligned
478  * @nbytes: bytes to copy
479  *
480  * Must handle nbytes < 8.
481  */
mid_copy_straight(struct pio_buf * pbuf,const void * from,size_t nbytes)482 static void mid_copy_straight(struct pio_buf *pbuf,
483 			      const void *from, size_t nbytes)
484 {
485 	void __iomem *dest = pbuf->start + (pbuf->qw_written * sizeof(u64));
486 	void __iomem *dend;			/* 8-byte data end */
487 
488 	/* calculate 8-byte data end */
489 	dend = dest + ((nbytes >> 3) * sizeof(u64));
490 
491 	if (pbuf->qw_written < PIO_BLOCK_QWS) {
492 		/*
493 		 * Still within SOP block.  We don't need to check for
494 		 * wrap because we are still in the first block and
495 		 * can only wrap on block boundaries.
496 		 */
497 		void __iomem *send;		/* SOP end */
498 		void __iomem *xend;
499 
500 		/*
501 		 * calculate the end of data or end of block, whichever
502 		 * comes first
503 		 */
504 		send = pbuf->start + PIO_BLOCK_SIZE;
505 		xend = min(send, dend);
506 
507 		/* shift up to SOP=1 space */
508 		dest += SOP_DISTANCE;
509 		xend += SOP_DISTANCE;
510 
511 		/* write 8-byte chunk data */
512 		while (dest < xend) {
513 			writeq(*(u64 *)from, dest);
514 			from += sizeof(u64);
515 			dest += sizeof(u64);
516 		}
517 
518 		/* shift down to SOP=0 space */
519 		dest -= SOP_DISTANCE;
520 	}
521 	/*
522 	 * At this point dest could be (either, both, or neither):
523 	 * - at dend
524 	 * - at the wrap
525 	 */
526 
527 	/*
528 	 * If the wrap comes before or matches the data end,
529 	 * copy until until the wrap, then wrap.
530 	 *
531 	 * If dest is at the wrap, we will fall into the if,
532 	 * not do the loop, when wrap.
533 	 *
534 	 * If the data ends at the end of the SOP above and
535 	 * the buffer wraps, then pbuf->end == dend == dest
536 	 * and nothing will get written.
537 	 */
538 	if (pbuf->end <= dend) {
539 		while (dest < pbuf->end) {
540 			writeq(*(u64 *)from, dest);
541 			from += sizeof(u64);
542 			dest += sizeof(u64);
543 		}
544 
545 		dest -= pbuf->sc->size;
546 		dend -= pbuf->sc->size;
547 	}
548 
549 	/* write 8-byte non-SOP, non-wrap chunk data */
550 	while (dest < dend) {
551 		writeq(*(u64 *)from, dest);
552 		from += sizeof(u64);
553 		dest += sizeof(u64);
554 	}
555 
556 	/* we know carry_bytes was zero on entry to this routine */
557 	read_low_bytes(pbuf, from, nbytes & 0x7);
558 
559 	pbuf->qw_written += nbytes >> 3;
560 }
561 
562 /*
563  * Segmented PIO Copy - middle
564  *
565  * Must handle any aligned tail and any aligned source with any byte count.
566  *
567  * @pbuf: a number of blocks allocated within a PIO send context
568  * @from: data source
569  * @nbytes: number of bytes to copy
570  */
seg_pio_copy_mid(struct pio_buf * pbuf,const void * from,size_t nbytes)571 void seg_pio_copy_mid(struct pio_buf *pbuf, const void *from, size_t nbytes)
572 {
573 	unsigned long from_align = (unsigned long)from & 0x7;
574 
575 	if (pbuf->carry_bytes + nbytes < 8) {
576 		/* not enough bytes to fill a QW */
577 		read_extra_bytes(pbuf, from, nbytes);
578 		return;
579 	}
580 
581 	if (from_align) {
582 		/* misaligned source pointer - align it */
583 		unsigned long to_align;
584 
585 		/* bytes to read to align "from" */
586 		to_align = 8 - from_align;
587 
588 		/*
589 		 * In the advance-to-alignment logic below, we do not need
590 		 * to check if we are using more than nbytes.  This is because
591 		 * if we are here, we already know that carry+nbytes will
592 		 * fill at least one QW.
593 		 */
594 		if (pbuf->carry_bytes + to_align < 8) {
595 			/* not enough align bytes to fill a QW */
596 			read_extra_bytes(pbuf, from, to_align);
597 			from += to_align;
598 			nbytes -= to_align;
599 		} else {
600 			/* bytes to fill carry */
601 			unsigned long to_fill = 8 - pbuf->carry_bytes;
602 			/* bytes left over to be read */
603 			unsigned long extra = to_align - to_fill;
604 			void __iomem *dest;
605 
606 			/* fill carry... */
607 			read_extra_bytes(pbuf, from, to_fill);
608 			from += to_fill;
609 			nbytes -= to_fill;
610 			/* may not be enough valid bytes left to align */
611 			if (extra > nbytes)
612 				extra = nbytes;
613 
614 			/* ...now write carry */
615 			dest = pbuf->start + (pbuf->qw_written * sizeof(u64));
616 
617 			/*
618 			 * The two checks immediately below cannot both be
619 			 * true, hence the else.  If we have wrapped, we
620 			 * cannot still be within the first block.
621 			 * Conversely, if we are still in the first block, we
622 			 * cannot have wrapped.  We do the wrap check first
623 			 * as that is more likely.
624 			 */
625 			/* adjust if we've wrapped */
626 			if (dest >= pbuf->end)
627 				dest -= pbuf->sc->size;
628 			/* jump to SOP range if within the first block */
629 			else if (pbuf->qw_written < PIO_BLOCK_QWS)
630 				dest += SOP_DISTANCE;
631 
632 			carry8_write8(pbuf->carry, dest);
633 			pbuf->qw_written++;
634 
635 			/* read any extra bytes to do final alignment */
636 			/* this will overwrite anything in pbuf->carry */
637 			read_low_bytes(pbuf, from, extra);
638 			from += extra;
639 			nbytes -= extra;
640 			/*
641 			 * If no bytes are left, return early - we are done.
642 			 * NOTE: This short-circuit is *required* because
643 			 * "extra" may have been reduced in size and "from"
644 			 * is not aligned, as required when leaving this
645 			 * if block.
646 			 */
647 			if (nbytes == 0)
648 				return;
649 		}
650 
651 		/* at this point, from is QW aligned */
652 	}
653 
654 	if (pbuf->carry_bytes)
655 		mid_copy_mix(pbuf, from, nbytes);
656 	else
657 		mid_copy_straight(pbuf, from, nbytes);
658 }
659 
660 /*
661  * Segmented PIO Copy - end
662  *
663  * Write any remainder (in pbuf->carry) and finish writing the whole block.
664  *
665  * @pbuf: a number of blocks allocated within a PIO send context
666  */
seg_pio_copy_end(struct pio_buf * pbuf)667 void seg_pio_copy_end(struct pio_buf *pbuf)
668 {
669 	void __iomem *dest = pbuf->start + (pbuf->qw_written * sizeof(u64));
670 
671 	/*
672 	 * The two checks immediately below cannot both be true, hence the
673 	 * else.  If we have wrapped, we cannot still be within the first
674 	 * block.  Conversely, if we are still in the first block, we
675 	 * cannot have wrapped.  We do the wrap check first as that is
676 	 * more likely.
677 	 */
678 	/* adjust if we have wrapped */
679 	if (dest >= pbuf->end)
680 		dest -= pbuf->sc->size;
681 	/* jump to the SOP range if within the first block */
682 	else if (pbuf->qw_written < PIO_BLOCK_QWS)
683 		dest += SOP_DISTANCE;
684 
685 	/* write final bytes, if any */
686 	if (carry_write8(pbuf, dest)) {
687 		dest += sizeof(u64);
688 		/*
689 		 * NOTE: We do not need to recalculate whether dest needs
690 		 * SOP_DISTANCE or not.
691 		 *
692 		 * If we are in the first block and the dangle write
693 		 * keeps us in the same block, dest will need
694 		 * to retain SOP_DISTANCE in the loop below.
695 		 *
696 		 * If we are in the first block and the dangle write pushes
697 		 * us to the next block, then loop below will not run
698 		 * and dest is not used.  Hence we do not need to update
699 		 * it.
700 		 *
701 		 * If we are past the first block, then SOP_DISTANCE
702 		 * was never added, so there is nothing to do.
703 		 */
704 	}
705 
706 	/* fill in rest of block */
707 	while (((unsigned long)dest & PIO_BLOCK_MASK) != 0) {
708 		writeq(0, dest);
709 		dest += sizeof(u64);
710 	}
711 
712 	/* finished with this buffer */
713 	this_cpu_dec(*pbuf->sc->buffers_allocated);
714 	preempt_enable();
715 }
716