1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * csum_partial_copy - do IP checksumming and copy
4  *
5  * (C) Copyright 1996 Linus Torvalds
6  * accelerated versions (and 21264 assembly versions ) contributed by
7  *	Rick Gorton	<rick.gorton@alpha-processor.com>
8  *
9  * Don't look at this too closely - you'll go mad. The things
10  * we do for performance..
11  */
12 
13 #include <linux/types.h>
14 #include <linux/string.h>
15 #include <linux/uaccess.h>
16 #include <net/checksum.h>
17 
18 
19 #define ldq_u(x,y) \
20 __asm__ __volatile__("ldq_u %0,%1":"=r" (x):"m" (*(const unsigned long *)(y)))
21 
22 #define stq_u(x,y) \
23 __asm__ __volatile__("stq_u %1,%0":"=m" (*(unsigned long *)(y)):"r" (x))
24 
25 #define extql(x,y,z) \
26 __asm__ __volatile__("extql %1,%2,%0":"=r" (z):"r" (x),"r" (y))
27 
28 #define extqh(x,y,z) \
29 __asm__ __volatile__("extqh %1,%2,%0":"=r" (z):"r" (x),"r" (y))
30 
31 #define mskql(x,y,z) \
32 __asm__ __volatile__("mskql %1,%2,%0":"=r" (z):"r" (x),"r" (y))
33 
34 #define mskqh(x,y,z) \
35 __asm__ __volatile__("mskqh %1,%2,%0":"=r" (z):"r" (x),"r" (y))
36 
37 #define insql(x,y,z) \
38 __asm__ __volatile__("insql %1,%2,%0":"=r" (z):"r" (x),"r" (y))
39 
40 #define insqh(x,y,z) \
41 __asm__ __volatile__("insqh %1,%2,%0":"=r" (z):"r" (x),"r" (y))
42 
43 #define __get_word(insn,x,ptr)				\
44 ({							\
45 	long __guu_err;					\
46 	__asm__ __volatile__(				\
47 	"1:	"#insn" %0,%2\n"			\
48 	"2:\n"						\
49 	EXC(1b,2b,%0,%1)				\
50 		: "=r"(x), "=r"(__guu_err)		\
51 		: "m"(__m(ptr)), "1"(0));		\
52 	__guu_err;					\
53 })
54 
from64to16(unsigned long x)55 static inline unsigned short from64to16(unsigned long x)
56 {
57 	/* Using extract instructions is a bit more efficient
58 	   than the original shift/bitmask version.  */
59 
60 	union {
61 		unsigned long	ul;
62 		unsigned int	ui[2];
63 		unsigned short	us[4];
64 	} in_v, tmp_v, out_v;
65 
66 	in_v.ul = x;
67 	tmp_v.ul = (unsigned long) in_v.ui[0] + (unsigned long) in_v.ui[1];
68 
69 	/* Since the bits of tmp_v.sh[3] are going to always be zero,
70 	   we don't have to bother to add that in.  */
71 	out_v.ul = (unsigned long) tmp_v.us[0] + (unsigned long) tmp_v.us[1]
72 			+ (unsigned long) tmp_v.us[2];
73 
74 	/* Similarly, out_v.us[2] is always zero for the final add.  */
75 	return out_v.us[0] + out_v.us[1];
76 }
77 
78 
79 
80 /*
81  * Ok. This isn't fun, but this is the EASY case.
82  */
83 static inline unsigned long
csum_partial_cfu_aligned(const unsigned long __user * src,unsigned long * dst,long len)84 csum_partial_cfu_aligned(const unsigned long __user *src, unsigned long *dst,
85 			 long len)
86 {
87 	unsigned long checksum = ~0U;
88 	unsigned long carry = 0;
89 
90 	while (len >= 0) {
91 		unsigned long word;
92 		if (__get_word(ldq, word, src))
93 			return 0;
94 		checksum += carry;
95 		src++;
96 		checksum += word;
97 		len -= 8;
98 		carry = checksum < word;
99 		*dst = word;
100 		dst++;
101 	}
102 	len += 8;
103 	checksum += carry;
104 	if (len) {
105 		unsigned long word, tmp;
106 		if (__get_word(ldq, word, src))
107 			return 0;
108 		tmp = *dst;
109 		mskql(word, len, word);
110 		checksum += word;
111 		mskqh(tmp, len, tmp);
112 		carry = checksum < word;
113 		*dst = word | tmp;
114 		checksum += carry;
115 	}
116 	return checksum;
117 }
118 
119 /*
120  * This is even less fun, but this is still reasonably
121  * easy.
122  */
123 static inline unsigned long
csum_partial_cfu_dest_aligned(const unsigned long __user * src,unsigned long * dst,unsigned long soff,long len)124 csum_partial_cfu_dest_aligned(const unsigned long __user *src,
125 			      unsigned long *dst,
126 			      unsigned long soff,
127 			      long len)
128 {
129 	unsigned long first;
130 	unsigned long word, carry;
131 	unsigned long lastsrc = 7+len+(unsigned long)src;
132 	unsigned long checksum = ~0U;
133 
134 	if (__get_word(ldq_u, first,src))
135 		return 0;
136 	carry = 0;
137 	while (len >= 0) {
138 		unsigned long second;
139 
140 		if (__get_word(ldq_u, second, src+1))
141 			return 0;
142 		extql(first, soff, word);
143 		len -= 8;
144 		src++;
145 		extqh(second, soff, first);
146 		checksum += carry;
147 		word |= first;
148 		first = second;
149 		checksum += word;
150 		*dst = word;
151 		dst++;
152 		carry = checksum < word;
153 	}
154 	len += 8;
155 	checksum += carry;
156 	if (len) {
157 		unsigned long tmp;
158 		unsigned long second;
159 		if (__get_word(ldq_u, second, lastsrc))
160 			return 0;
161 		tmp = *dst;
162 		extql(first, soff, word);
163 		extqh(second, soff, first);
164 		word |= first;
165 		mskql(word, len, word);
166 		checksum += word;
167 		mskqh(tmp, len, tmp);
168 		carry = checksum < word;
169 		*dst = word | tmp;
170 		checksum += carry;
171 	}
172 	return checksum;
173 }
174 
175 /*
176  * This is slightly less fun than the above..
177  */
178 static inline unsigned long
csum_partial_cfu_src_aligned(const unsigned long __user * src,unsigned long * dst,unsigned long doff,long len,unsigned long partial_dest)179 csum_partial_cfu_src_aligned(const unsigned long __user *src,
180 			     unsigned long *dst,
181 			     unsigned long doff,
182 			     long len,
183 			     unsigned long partial_dest)
184 {
185 	unsigned long carry = 0;
186 	unsigned long word;
187 	unsigned long second_dest;
188 	unsigned long checksum = ~0U;
189 
190 	mskql(partial_dest, doff, partial_dest);
191 	while (len >= 0) {
192 		if (__get_word(ldq, word, src))
193 			return 0;
194 		len -= 8;
195 		insql(word, doff, second_dest);
196 		checksum += carry;
197 		stq_u(partial_dest | second_dest, dst);
198 		src++;
199 		checksum += word;
200 		insqh(word, doff, partial_dest);
201 		carry = checksum < word;
202 		dst++;
203 	}
204 	len += 8;
205 	if (len) {
206 		checksum += carry;
207 		if (__get_word(ldq, word, src))
208 			return 0;
209 		mskql(word, len, word);
210 		len -= 8;
211 		checksum += word;
212 		insql(word, doff, second_dest);
213 		len += doff;
214 		carry = checksum < word;
215 		partial_dest |= second_dest;
216 		if (len >= 0) {
217 			stq_u(partial_dest, dst);
218 			if (!len) goto out;
219 			dst++;
220 			insqh(word, doff, partial_dest);
221 		}
222 		doff = len;
223 	}
224 	ldq_u(second_dest, dst);
225 	mskqh(second_dest, doff, second_dest);
226 	stq_u(partial_dest | second_dest, dst);
227 out:
228 	checksum += carry;
229 	return checksum;
230 }
231 
232 /*
233  * This is so totally un-fun that it's frightening. Don't
234  * look at this too closely, you'll go blind.
235  */
236 static inline unsigned long
csum_partial_cfu_unaligned(const unsigned long __user * src,unsigned long * dst,unsigned long soff,unsigned long doff,long len,unsigned long partial_dest)237 csum_partial_cfu_unaligned(const unsigned long __user * src,
238 			   unsigned long * dst,
239 			   unsigned long soff, unsigned long doff,
240 			   long len, unsigned long partial_dest)
241 {
242 	unsigned long carry = 0;
243 	unsigned long first;
244 	unsigned long lastsrc;
245 	unsigned long checksum = ~0U;
246 
247 	if (__get_word(ldq_u, first, src))
248 		return 0;
249 	lastsrc = 7+len+(unsigned long)src;
250 	mskql(partial_dest, doff, partial_dest);
251 	while (len >= 0) {
252 		unsigned long second, word;
253 		unsigned long second_dest;
254 
255 		if (__get_word(ldq_u, second, src+1))
256 			return 0;
257 		extql(first, soff, word);
258 		checksum += carry;
259 		len -= 8;
260 		extqh(second, soff, first);
261 		src++;
262 		word |= first;
263 		first = second;
264 		insql(word, doff, second_dest);
265 		checksum += word;
266 		stq_u(partial_dest | second_dest, dst);
267 		carry = checksum < word;
268 		insqh(word, doff, partial_dest);
269 		dst++;
270 	}
271 	len += doff;
272 	checksum += carry;
273 	if (len >= 0) {
274 		unsigned long second, word;
275 		unsigned long second_dest;
276 
277 		if (__get_word(ldq_u, second, lastsrc))
278 			return 0;
279 		extql(first, soff, word);
280 		extqh(second, soff, first);
281 		word |= first;
282 		first = second;
283 		mskql(word, len-doff, word);
284 		checksum += word;
285 		insql(word, doff, second_dest);
286 		carry = checksum < word;
287 		stq_u(partial_dest | second_dest, dst);
288 		if (len) {
289 			ldq_u(second_dest, dst+1);
290 			insqh(word, doff, partial_dest);
291 			mskqh(second_dest, len, second_dest);
292 			stq_u(partial_dest | second_dest, dst+1);
293 		}
294 		checksum += carry;
295 	} else {
296 		unsigned long second, word;
297 		unsigned long second_dest;
298 
299 		if (__get_word(ldq_u, second, lastsrc))
300 			return 0;
301 		extql(first, soff, word);
302 		extqh(second, soff, first);
303 		word |= first;
304 		ldq_u(second_dest, dst);
305 		mskql(word, len-doff, word);
306 		checksum += word;
307 		mskqh(second_dest, len, second_dest);
308 		carry = checksum < word;
309 		insql(word, doff, word);
310 		stq_u(partial_dest | word | second_dest, dst);
311 		checksum += carry;
312 	}
313 	return checksum;
314 }
315 
__csum_and_copy(const void __user * src,void * dst,int len)316 static __wsum __csum_and_copy(const void __user *src, void *dst, int len)
317 {
318 	unsigned long soff = 7 & (unsigned long) src;
319 	unsigned long doff = 7 & (unsigned long) dst;
320 	unsigned long checksum;
321 
322 	if (!doff) {
323 		if (!soff)
324 			checksum = csum_partial_cfu_aligned(
325 				(const unsigned long __user *) src,
326 				(unsigned long *) dst, len-8);
327 		else
328 			checksum = csum_partial_cfu_dest_aligned(
329 				(const unsigned long __user *) src,
330 				(unsigned long *) dst,
331 				soff, len-8);
332 	} else {
333 		unsigned long partial_dest;
334 		ldq_u(partial_dest, dst);
335 		if (!soff)
336 			checksum = csum_partial_cfu_src_aligned(
337 				(const unsigned long __user *) src,
338 				(unsigned long *) dst,
339 				doff, len-8, partial_dest);
340 		else
341 			checksum = csum_partial_cfu_unaligned(
342 				(const unsigned long __user *) src,
343 				(unsigned long *) dst,
344 				soff, doff, len-8, partial_dest);
345 	}
346 	return (__force __wsum)from64to16 (checksum);
347 }
348 
349 __wsum
csum_and_copy_from_user(const void __user * src,void * dst,int len)350 csum_and_copy_from_user(const void __user *src, void *dst, int len)
351 {
352 	if (!access_ok(src, len))
353 		return 0;
354 	return __csum_and_copy(src, dst, len);
355 }
356 
357 __wsum
csum_partial_copy_nocheck(const void * src,void * dst,int len)358 csum_partial_copy_nocheck(const void *src, void *dst, int len)
359 {
360 	return __csum_and_copy((__force const void __user *)src,
361 						dst, len);
362 }
363 EXPORT_SYMBOL(csum_partial_copy_nocheck);
364