1 // SPDX-License-Identifier: GPL-2.0 or BSD-3-Clause
8 /* additive distance between non-SOP and SOP space */
10 #define PIO_BLOCK_MASK (PIO_BLOCK_SIZE - 1)
11 /* number of QUADWORDs in a block */
15  * pio_copy - copy data block to MMIO space
20  * @count: number of DWORD (32-bit) quantities to copy from source
22  * Copy data from source to PIO Send Buffer memory, 8 bytes at a time.
23  * Must always write full BLOCK_SIZE bytes blocks.  The first block must
27  * o pbuf->start always starts on a block boundary
28  * o pbuf can wrap only at a block boundary
33 	void __iomem *dest = pbuf->start + SOP_DISTANCE;  in pio_copy()
35 	void __iomem *dend;			/* 8-byte data end */  in pio_copy()
41 	/* calculate where the QWORD data ends - in SOP=1 space */  in pio_copy()
46 		 * all QWORD data is within the SOP block, does *not*  in pio_copy()
47 		 * reach the end of the SOP block  in pio_copy()
57 		 * 0. We're not on the SOP block boundary  in pio_copy()
59 		 *    the SOP block  in pio_copy()
60 		 * 2. We cannot wrap except on a block boundary.  in pio_copy()
63 		/* QWORD data extends _to_ or beyond the SOP block */  in pio_copy()
65 		/* write 8-byte SOP chunk data */  in pio_copy()
72 		dest -= SOP_DISTANCE;  in pio_copy()
73 		dend -= SOP_DISTANCE;  in pio_copy()
77 		 * copy until until the wrap, then wrap.  in pio_copy()
80 		 * the buffer wraps, then pbuf->end == dend == dest  in pio_copy()
84 		if (pbuf->end <= dend) {  in pio_copy()
85 			while (dest < pbuf->end) {  in pio_copy()
91 			dest -= pbuf->sc->size;  in pio_copy()
92 			dend -= pbuf->sc->size;  in pio_copy()
95 		/* write 8-byte non-SOP, non-wrap chunk data */  in pio_copy()
114 	 * fill in rest of block, no need to check pbuf->end  in pio_copy()
115 	 * as we only wrap on a block boundary  in pio_copy()
123 	this_cpu_dec(*pbuf->sc->buffers_allocated);  in pio_copy()
134  * "zero" shift - bit shift used to zero out upper bytes.  Input is
137 #define zshift(x) (8 * (8 - (x)))
140  * "merge" shift - bit shift used to merge with carry bytes.  Input is
146  * Jump copy - no-loop copy for < 8 bytes.
176  * of pbuf->carry.  Other bytes are left as-is.  Any previous
177  * value in pbuf->carry is lost.
186 	pbuf->carry.val64 = 0;  in read_low_bytes()
187 	jcopy(&pbuf->carry.val8[0], from, nbytes);  in read_low_bytes()
188 	pbuf->carry_bytes = nbytes;  in read_low_bytes()
192  * Read nbytes bytes from "from" and put them at the end of pbuf->carry.
202 	jcopy(&pbuf->carry.val8[pbuf->carry_bytes], from, nbytes);  in read_extra_bytes()
203 	pbuf->carry_bytes += nbytes;  in read_extra_bytes()
207  * Write a quad word using parts of pbuf->carry and the next 8 bytes of src.
209  * pbuf->carry with the upper bytes zeroed..
223 	temp = pbuf->carry.val64 | (new << mshift(pbuf->carry_bytes));  in merge_write8()
225 	pbuf->carry.val64 = new >> zshift(pbuf->carry_bytes);  in merge_write8()
239  * Returns 0 on nothing written, non-zero on quad word written.
243 	if (pbuf->carry_bytes) {  in carry_write8()
245 		writeq(pbuf->carry.val64, dest);  in carry_write8()
253  * Segmented PIO Copy - start
255  * Start a PIO copy.
260  * @nbytes: bytes to copy
265 	void __iomem *dest = pbuf->start + SOP_DISTANCE;  in seg_pio_copy_start()
267 	void __iomem *dend;			/* 8-byte data end */  in seg_pio_copy_start()
272 	/* calculate where the QWORD data ends - in SOP=1 space */  in seg_pio_copy_start()
277 		 * all QWORD data is within the SOP block, does *not*  in seg_pio_copy_start()
278 		 * reach the end of the SOP block  in seg_pio_copy_start()
288 		 * 0. We're not on the SOP block boundary  in seg_pio_copy_start()
290 		 *    the SOP block  in seg_pio_copy_start()
291 		 * 2. We cannot wrap except on a block boundary.  in seg_pio_copy_start()
294 		/* QWORD data extends _to_ or beyond the SOP block */  in seg_pio_copy_start()
296 		/* write 8-byte SOP chunk data */  in seg_pio_copy_start()
303 		dest -= SOP_DISTANCE;  in seg_pio_copy_start()
304 		dend -= SOP_DISTANCE;  in seg_pio_copy_start()
308 		 * copy until until the wrap, then wrap.  in seg_pio_copy_start()
311 		 * the buffer wraps, then pbuf->end == dend == dest  in seg_pio_copy_start()
315 		if (pbuf->end <= dend) {  in seg_pio_copy_start()
316 			while (dest < pbuf->end) {  in seg_pio_copy_start()
322 			dest -= pbuf->sc->size;  in seg_pio_copy_start()
323 			dend -= pbuf->sc->size;  in seg_pio_copy_start()
326 		/* write 8-byte non-SOP, non-wrap chunk data */  in seg_pio_copy_start()
340 	pbuf->qw_written = 1 /*PBC*/ + (nbytes >> 3);  in seg_pio_copy_start()
344  * Mid copy helper, "mixed case" - source is 64-bit aligned but carry
345  * bytes are non-zero.
351  * @nbytes: bytes to copy
357 	void __iomem *dest = pbuf->start + (pbuf->qw_written * sizeof(u64));  in mid_copy_mix()
358 	void __iomem *dend;			/* 8-byte data end */  in mid_copy_mix()
362 	/* calculate 8-byte data end */  in mid_copy_mix()
365 	if (pbuf->qw_written < PIO_BLOCK_QWS) {  in mid_copy_mix()
367 		 * Still within SOP block.  We don't need to check for  in mid_copy_mix()
368 		 * wrap because we are still in the first block and  in mid_copy_mix()
369 		 * can only wrap on block boundaries.  in mid_copy_mix()
375 		 * calculate the end of data or end of block, whichever  in mid_copy_mix()
378 		send = pbuf->start + PIO_BLOCK_SIZE;  in mid_copy_mix()
385 		/* write 8-byte chunk data */  in mid_copy_mix()
393 		dest -= SOP_DISTANCE;  in mid_copy_mix()
397 	 * - at dend  in mid_copy_mix()
398 	 * - at the wrap  in mid_copy_mix()
403 	 * copy until until the wrap, then wrap.  in mid_copy_mix()
409 	 * the buffer wraps, then pbuf->end == dend == dest  in mid_copy_mix()
412 	if (pbuf->end <= dend) {  in mid_copy_mix()
413 		while (dest < pbuf->end) {  in mid_copy_mix()
419 		dest -= pbuf->sc->size;  in mid_copy_mix()
420 		dend -= pbuf->sc->size;  in mid_copy_mix()
423 	/* write 8-byte non-SOP, non-wrap chunk data */  in mid_copy_mix()
430 	pbuf->qw_written += qw_to_write;  in mid_copy_mix()
432 	/* handle carry and left-over bytes */  in mid_copy_mix()
433 	if (pbuf->carry_bytes + bytes_left >= 8) {  in mid_copy_mix()
436 		/* there is enough to fill another qw - fill carry */  in mid_copy_mix()
437 		nread = 8 - pbuf->carry_bytes;  in mid_copy_mix()
441 		 * One more write - but need to make sure dest is correct.  in mid_copy_mix()
447 		 * first block. Conversely, if we are still in the first block,  in mid_copy_mix()
452 		if (dest >= pbuf->end)  in mid_copy_mix()
453 			dest -= pbuf->sc->size;  in mid_copy_mix()
454 		/* jump to the SOP range if within the first block */  in mid_copy_mix()
455 		else if (pbuf->qw_written < PIO_BLOCK_QWS)  in mid_copy_mix()
459 		carry8_write8(pbuf->carry, dest);  in mid_copy_mix()
460 		pbuf->qw_written++;  in mid_copy_mix()
463 		bytes_left -= nread;  in mid_copy_mix()
473  * Mid copy helper, "straight case" - source pointer is 64-bit aligned
478  * @nbytes: bytes to copy
485 	void __iomem *dest = pbuf->start + (pbuf->qw_written * sizeof(u64));  in mid_copy_straight()
486 	void __iomem *dend;			/* 8-byte data end */  in mid_copy_straight()
488 	/* calculate 8-byte data end */  in mid_copy_straight()
491 	if (pbuf->qw_written < PIO_BLOCK_QWS) {  in mid_copy_straight()
493 		 * Still within SOP block.  We don't need to check for  in mid_copy_straight()
494 		 * wrap because we are still in the first block and  in mid_copy_straight()
495 		 * can only wrap on block boundaries.  in mid_copy_straight()
501 		 * calculate the end of data or end of block, whichever  in mid_copy_straight()
504 		send = pbuf->start + PIO_BLOCK_SIZE;  in mid_copy_straight()
511 		/* write 8-byte chunk data */  in mid_copy_straight()
519 		dest -= SOP_DISTANCE;  in mid_copy_straight()
523 	 * - at dend  in mid_copy_straight()
524 	 * - at the wrap  in mid_copy_straight()
529 	 * copy until until the wrap, then wrap.  in mid_copy_straight()
535 	 * the buffer wraps, then pbuf->end == dend == dest  in mid_copy_straight()
538 	if (pbuf->end <= dend) {  in mid_copy_straight()
539 		while (dest < pbuf->end) {  in mid_copy_straight()
545 		dest -= pbuf->sc->size;  in mid_copy_straight()
546 		dend -= pbuf->sc->size;  in mid_copy_straight()
549 	/* write 8-byte non-SOP, non-wrap chunk data */  in mid_copy_straight()
559 	pbuf->qw_written += nbytes >> 3;  in mid_copy_straight()
563  * Segmented PIO Copy - middle
569  * @nbytes: number of bytes to copy
575 	if (pbuf->carry_bytes + nbytes < 8) {  in seg_pio_copy_mid()
582 		/* misaligned source pointer - align it */  in seg_pio_copy_mid()
586 		to_align = 8 - from_align;  in seg_pio_copy_mid()
589 		 * In the advance-to-alignment logic below, we do not need  in seg_pio_copy_mid()
594 		if (pbuf->carry_bytes + to_align < 8) {  in seg_pio_copy_mid()
598 			nbytes -= to_align;  in seg_pio_copy_mid()
601 			unsigned long to_fill = 8 - pbuf->carry_bytes;  in seg_pio_copy_mid()
603 			unsigned long extra = to_align - to_fill;  in seg_pio_copy_mid()
609 			nbytes -= to_fill;  in seg_pio_copy_mid()
615 			dest = pbuf->start + (pbuf->qw_written * sizeof(u64));  in seg_pio_copy_mid()
620 			 * cannot still be within the first block.  in seg_pio_copy_mid()
621 			 * Conversely, if we are still in the first block, we  in seg_pio_copy_mid()
626 			if (dest >= pbuf->end)  in seg_pio_copy_mid()
627 				dest -= pbuf->sc->size;  in seg_pio_copy_mid()
628 			/* jump to SOP range if within the first block */  in seg_pio_copy_mid()
629 			else if (pbuf->qw_written < PIO_BLOCK_QWS)  in seg_pio_copy_mid()
632 			carry8_write8(pbuf->carry, dest);  in seg_pio_copy_mid()
633 			pbuf->qw_written++;  in seg_pio_copy_mid()
636 			/* this will overwrite anything in pbuf->carry */  in seg_pio_copy_mid()
639 			nbytes -= extra;  in seg_pio_copy_mid()
641 			 * If no bytes are left, return early - we are done.  in seg_pio_copy_mid()
642 			 * NOTE: This short-circuit is *required* because  in seg_pio_copy_mid()
645 			 * if block.  in seg_pio_copy_mid()
654 	if (pbuf->carry_bytes)  in seg_pio_copy_mid()
661  * Segmented PIO Copy - end
663  * Write any remainder (in pbuf->carry) and finish writing the whole block.
669 	void __iomem *dest = pbuf->start + (pbuf->qw_written * sizeof(u64));  in seg_pio_copy_end()
674 	 * block.  Conversely, if we are still in the first block, we  in seg_pio_copy_end()
679 	if (dest >= pbuf->end)  in seg_pio_copy_end()
680 		dest -= pbuf->sc->size;  in seg_pio_copy_end()
681 	/* jump to the SOP range if within the first block */  in seg_pio_copy_end()
682 	else if (pbuf->qw_written < PIO_BLOCK_QWS)  in seg_pio_copy_end()
692 		 * If we are in the first block and the dangle write  in seg_pio_copy_end()
693 		 * keeps us in the same block, dest will need  in seg_pio_copy_end()
696 		 * If we are in the first block and the dangle write pushes  in seg_pio_copy_end()
697 		 * us to the next block, then loop below will not run  in seg_pio_copy_end()
701 		 * If we are past the first block, then SOP_DISTANCE  in seg_pio_copy_end()
706 	/* fill in rest of block */  in seg_pio_copy_end()
713 	this_cpu_dec(*pbuf->sc->buffers_allocated);  in seg_pio_copy_end()