1 // SPDX-License-Identifier: GPL-2.0-only
2 /* The industrial I/O core
3  *
4  * Copyright (c) 2008 Jonathan Cameron
5  *
6  * Handling of buffer allocation / resizing.
7  *
8  * Things to look at here.
9  * - Better memory allocation techniques?
10  * - Alternative access techniques?
11  */
12 #include <linux/kernel.h>
13 #include <linux/export.h>
14 #include <linux/device.h>
15 #include <linux/fs.h>
16 #include <linux/cdev.h>
17 #include <linux/slab.h>
18 #include <linux/poll.h>
19 #include <linux/sched/signal.h>
20 
21 #include <linux/iio/iio.h>
22 #include "iio_core.h"
23 #include <linux/iio/sysfs.h>
24 #include <linux/iio/buffer.h>
25 #include <linux/iio/buffer_impl.h>
26 
27 static const char * const iio_endian_prefix[] = {
28 	[IIO_BE] = "be",
29 	[IIO_LE] = "le",
30 };
31 
32 static bool iio_buffer_is_active(struct iio_buffer *buf)
33 {
34 	return !list_empty(&buf->buffer_list);
35 }
36 
37 static size_t iio_buffer_data_available(struct iio_buffer *buf)
38 {
39 	return buf->access->data_available(buf);
40 }
41 
42 static int iio_buffer_flush_hwfifo(struct iio_dev *indio_dev,
43 				   struct iio_buffer *buf, size_t required)
44 {
45 	if (!indio_dev->info->hwfifo_flush_to_buffer)
46 		return -ENODEV;
47 
48 	return indio_dev->info->hwfifo_flush_to_buffer(indio_dev, required);
49 }
50 
51 static bool iio_buffer_ready(struct iio_dev *indio_dev, struct iio_buffer *buf,
52 			     size_t to_wait, int to_flush)
53 {
54 	size_t avail;
55 	int flushed = 0;
56 
57 	/* wakeup if the device was unregistered */
58 	if (!indio_dev->info)
59 		return true;
60 
61 	/* drain the buffer if it was disabled */
62 	if (!iio_buffer_is_active(buf)) {
63 		to_wait = min_t(size_t, to_wait, 1);
64 		to_flush = 0;
65 	}
66 
67 	avail = iio_buffer_data_available(buf);
68 
69 	if (avail >= to_wait) {
70 		/* force a flush for non-blocking reads */
71 		if (!to_wait && avail < to_flush)
72 			iio_buffer_flush_hwfifo(indio_dev, buf,
73 						to_flush - avail);
74 		return true;
75 	}
76 
77 	if (to_flush)
78 		flushed = iio_buffer_flush_hwfifo(indio_dev, buf,
79 						  to_wait - avail);
80 	if (flushed <= 0)
81 		return false;
82 
83 	if (avail + flushed >= to_wait)
84 		return true;
85 
86 	return false;
87 }
88 
89 /**
90  * iio_buffer_read_outer() - chrdev read for buffer access
91  * @filp:	File structure pointer for the char device
92  * @buf:	Destination buffer for iio buffer read
93  * @n:		First n bytes to read
94  * @f_ps:	Long offset provided by the user as a seek position
95  *
96  * This function relies on all buffer implementations having an
97  * iio_buffer as their first element.
98  *
99  * Return: negative values corresponding to error codes or ret != 0
100  *	   for ending the reading activity
101  **/
102 ssize_t iio_buffer_read_outer(struct file *filp, char __user *buf,
103 			      size_t n, loff_t *f_ps)
104 {
105 	struct iio_dev *indio_dev = filp->private_data;
106 	struct iio_buffer *rb = indio_dev->buffer;
107 	DEFINE_WAIT_FUNC(wait, woken_wake_function);
108 	size_t datum_size;
109 	size_t to_wait;
110 	int ret = 0;
111 
112 	if (!indio_dev->info)
113 		return -ENODEV;
114 
115 	if (!rb || !rb->access->read)
116 		return -EINVAL;
117 
118 	datum_size = rb->bytes_per_datum;
119 
120 	/*
121 	 * If datum_size is 0 there will never be anything to read from the
122 	 * buffer, so signal end of file now.
123 	 */
124 	if (!datum_size)
125 		return 0;
126 
127 	if (filp->f_flags & O_NONBLOCK)
128 		to_wait = 0;
129 	else
130 		to_wait = min_t(size_t, n / datum_size, rb->watermark);
131 
132 	add_wait_queue(&rb->pollq, &wait);
133 	do {
134 		if (!indio_dev->info) {
135 			ret = -ENODEV;
136 			break;
137 		}
138 
139 		if (!iio_buffer_ready(indio_dev, rb, to_wait, n / datum_size)) {
140 			if (signal_pending(current)) {
141 				ret = -ERESTARTSYS;
142 				break;
143 			}
144 
145 			wait_woken(&wait, TASK_INTERRUPTIBLE,
146 				   MAX_SCHEDULE_TIMEOUT);
147 			continue;
148 		}
149 
150 		ret = rb->access->read(rb, n, buf);
151 		if (ret == 0 && (filp->f_flags & O_NONBLOCK))
152 			ret = -EAGAIN;
153 	} while (ret == 0);
154 	remove_wait_queue(&rb->pollq, &wait);
155 
156 	return ret;
157 }
158 
159 /**
160  * iio_buffer_poll() - poll the buffer to find out if it has data
161  * @filp:	File structure pointer for device access
162  * @wait:	Poll table structure pointer for which the driver adds
163  *		a wait queue
164  *
165  * Return: (EPOLLIN | EPOLLRDNORM) if data is available for reading
166  *	   or 0 for other cases
167  */
168 __poll_t iio_buffer_poll(struct file *filp,
169 			     struct poll_table_struct *wait)
170 {
171 	struct iio_dev *indio_dev = filp->private_data;
172 	struct iio_buffer *rb = indio_dev->buffer;
173 
174 	if (!indio_dev->info || rb == NULL)
175 		return 0;
176 
177 	poll_wait(filp, &rb->pollq, wait);
178 	if (iio_buffer_ready(indio_dev, rb, rb->watermark, 0))
179 		return EPOLLIN | EPOLLRDNORM;
180 	return 0;
181 }
182 
183 /**
184  * iio_buffer_wakeup_poll - Wakes up the buffer waitqueue
185  * @indio_dev: The IIO device
186  *
187  * Wakes up the event waitqueue used for poll(). Should usually
188  * be called when the device is unregistered.
189  */
190 void iio_buffer_wakeup_poll(struct iio_dev *indio_dev)
191 {
192 	if (!indio_dev->buffer)
193 		return;
194 
195 	wake_up(&indio_dev->buffer->pollq);
196 }
197 
198 void iio_buffer_init(struct iio_buffer *buffer)
199 {
200 	INIT_LIST_HEAD(&buffer->demux_list);
201 	INIT_LIST_HEAD(&buffer->buffer_list);
202 	init_waitqueue_head(&buffer->pollq);
203 	kref_init(&buffer->ref);
204 	if (!buffer->watermark)
205 		buffer->watermark = 1;
206 }
207 EXPORT_SYMBOL(iio_buffer_init);
208 
209 /**
210  * iio_buffer_set_attrs - Set buffer specific attributes
211  * @buffer: The buffer for which we are setting attributes
212  * @attrs: Pointer to a null terminated list of pointers to attributes
213  */
214 void iio_buffer_set_attrs(struct iio_buffer *buffer,
215 			 const struct attribute **attrs)
216 {
217 	buffer->attrs = attrs;
218 }
219 EXPORT_SYMBOL_GPL(iio_buffer_set_attrs);
220 
221 static ssize_t iio_show_scan_index(struct device *dev,
222 				   struct device_attribute *attr,
223 				   char *buf)
224 {
225 	return sprintf(buf, "%u\n", to_iio_dev_attr(attr)->c->scan_index);
226 }
227 
228 static ssize_t iio_show_fixed_type(struct device *dev,
229 				   struct device_attribute *attr,
230 				   char *buf)
231 {
232 	struct iio_dev_attr *this_attr = to_iio_dev_attr(attr);
233 	u8 type = this_attr->c->scan_type.endianness;
234 
235 	if (type == IIO_CPU) {
236 #ifdef __LITTLE_ENDIAN
237 		type = IIO_LE;
238 #else
239 		type = IIO_BE;
240 #endif
241 	}
242 	if (this_attr->c->scan_type.repeat > 1)
243 		return sprintf(buf, "%s:%c%d/%dX%d>>%u\n",
244 		       iio_endian_prefix[type],
245 		       this_attr->c->scan_type.sign,
246 		       this_attr->c->scan_type.realbits,
247 		       this_attr->c->scan_type.storagebits,
248 		       this_attr->c->scan_type.repeat,
249 		       this_attr->c->scan_type.shift);
250 		else
251 			return sprintf(buf, "%s:%c%d/%d>>%u\n",
252 		       iio_endian_prefix[type],
253 		       this_attr->c->scan_type.sign,
254 		       this_attr->c->scan_type.realbits,
255 		       this_attr->c->scan_type.storagebits,
256 		       this_attr->c->scan_type.shift);
257 }
258 
259 static ssize_t iio_scan_el_show(struct device *dev,
260 				struct device_attribute *attr,
261 				char *buf)
262 {
263 	int ret;
264 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
265 
266 	/* Ensure ret is 0 or 1. */
267 	ret = !!test_bit(to_iio_dev_attr(attr)->address,
268 		       indio_dev->buffer->scan_mask);
269 
270 	return sprintf(buf, "%d\n", ret);
271 }
272 
273 /* Note NULL used as error indicator as it doesn't make sense. */
274 static const unsigned long *iio_scan_mask_match(const unsigned long *av_masks,
275 					  unsigned int masklength,
276 					  const unsigned long *mask,
277 					  bool strict)
278 {
279 	if (bitmap_empty(mask, masklength))
280 		return NULL;
281 	while (*av_masks) {
282 		if (strict) {
283 			if (bitmap_equal(mask, av_masks, masklength))
284 				return av_masks;
285 		} else {
286 			if (bitmap_subset(mask, av_masks, masklength))
287 				return av_masks;
288 		}
289 		av_masks += BITS_TO_LONGS(masklength);
290 	}
291 	return NULL;
292 }
293 
294 static bool iio_validate_scan_mask(struct iio_dev *indio_dev,
295 	const unsigned long *mask)
296 {
297 	if (!indio_dev->setup_ops->validate_scan_mask)
298 		return true;
299 
300 	return indio_dev->setup_ops->validate_scan_mask(indio_dev, mask);
301 }
302 
303 /**
304  * iio_scan_mask_set() - set particular bit in the scan mask
305  * @indio_dev: the iio device
306  * @buffer: the buffer whose scan mask we are interested in
307  * @bit: the bit to be set.
308  *
309  * Note that at this point we have no way of knowing what other
310  * buffers might request, hence this code only verifies that the
311  * individual buffers request is plausible.
312  */
313 static int iio_scan_mask_set(struct iio_dev *indio_dev,
314 		      struct iio_buffer *buffer, int bit)
315 {
316 	const unsigned long *mask;
317 	unsigned long *trialmask;
318 
319 	trialmask = kcalloc(BITS_TO_LONGS(indio_dev->masklength),
320 			    sizeof(*trialmask), GFP_KERNEL);
321 	if (trialmask == NULL)
322 		return -ENOMEM;
323 	if (!indio_dev->masklength) {
324 		WARN(1, "Trying to set scanmask prior to registering buffer\n");
325 		goto err_invalid_mask;
326 	}
327 	bitmap_copy(trialmask, buffer->scan_mask, indio_dev->masklength);
328 	set_bit(bit, trialmask);
329 
330 	if (!iio_validate_scan_mask(indio_dev, trialmask))
331 		goto err_invalid_mask;
332 
333 	if (indio_dev->available_scan_masks) {
334 		mask = iio_scan_mask_match(indio_dev->available_scan_masks,
335 					   indio_dev->masklength,
336 					   trialmask, false);
337 		if (!mask)
338 			goto err_invalid_mask;
339 	}
340 	bitmap_copy(buffer->scan_mask, trialmask, indio_dev->masklength);
341 
342 	bitmap_free(trialmask);
343 
344 	return 0;
345 
346 err_invalid_mask:
347 	bitmap_free(trialmask);
348 	return -EINVAL;
349 }
350 
351 static int iio_scan_mask_clear(struct iio_buffer *buffer, int bit)
352 {
353 	clear_bit(bit, buffer->scan_mask);
354 	return 0;
355 }
356 
357 static int iio_scan_mask_query(struct iio_dev *indio_dev,
358 			       struct iio_buffer *buffer, int bit)
359 {
360 	if (bit > indio_dev->masklength)
361 		return -EINVAL;
362 
363 	if (!buffer->scan_mask)
364 		return 0;
365 
366 	/* Ensure return value is 0 or 1. */
367 	return !!test_bit(bit, buffer->scan_mask);
368 };
369 
370 static ssize_t iio_scan_el_store(struct device *dev,
371 				 struct device_attribute *attr,
372 				 const char *buf,
373 				 size_t len)
374 {
375 	int ret;
376 	bool state;
377 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
378 	struct iio_buffer *buffer = indio_dev->buffer;
379 	struct iio_dev_attr *this_attr = to_iio_dev_attr(attr);
380 
381 	ret = strtobool(buf, &state);
382 	if (ret < 0)
383 		return ret;
384 	mutex_lock(&indio_dev->mlock);
385 	if (iio_buffer_is_active(indio_dev->buffer)) {
386 		ret = -EBUSY;
387 		goto error_ret;
388 	}
389 	ret = iio_scan_mask_query(indio_dev, buffer, this_attr->address);
390 	if (ret < 0)
391 		goto error_ret;
392 	if (!state && ret) {
393 		ret = iio_scan_mask_clear(buffer, this_attr->address);
394 		if (ret)
395 			goto error_ret;
396 	} else if (state && !ret) {
397 		ret = iio_scan_mask_set(indio_dev, buffer, this_attr->address);
398 		if (ret)
399 			goto error_ret;
400 	}
401 
402 error_ret:
403 	mutex_unlock(&indio_dev->mlock);
404 
405 	return ret < 0 ? ret : len;
406 
407 }
408 
409 static ssize_t iio_scan_el_ts_show(struct device *dev,
410 				   struct device_attribute *attr,
411 				   char *buf)
412 {
413 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
414 	return sprintf(buf, "%d\n", indio_dev->buffer->scan_timestamp);
415 }
416 
417 static ssize_t iio_scan_el_ts_store(struct device *dev,
418 				    struct device_attribute *attr,
419 				    const char *buf,
420 				    size_t len)
421 {
422 	int ret;
423 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
424 	bool state;
425 
426 	ret = strtobool(buf, &state);
427 	if (ret < 0)
428 		return ret;
429 
430 	mutex_lock(&indio_dev->mlock);
431 	if (iio_buffer_is_active(indio_dev->buffer)) {
432 		ret = -EBUSY;
433 		goto error_ret;
434 	}
435 	indio_dev->buffer->scan_timestamp = state;
436 error_ret:
437 	mutex_unlock(&indio_dev->mlock);
438 
439 	return ret ? ret : len;
440 }
441 
442 static int iio_buffer_add_channel_sysfs(struct iio_dev *indio_dev,
443 					const struct iio_chan_spec *chan)
444 {
445 	int ret, attrcount = 0;
446 	struct iio_buffer *buffer = indio_dev->buffer;
447 
448 	ret = __iio_add_chan_devattr("index",
449 				     chan,
450 				     &iio_show_scan_index,
451 				     NULL,
452 				     0,
453 				     IIO_SEPARATE,
454 				     &indio_dev->dev,
455 				     &buffer->scan_el_dev_attr_list);
456 	if (ret)
457 		return ret;
458 	attrcount++;
459 	ret = __iio_add_chan_devattr("type",
460 				     chan,
461 				     &iio_show_fixed_type,
462 				     NULL,
463 				     0,
464 				     0,
465 				     &indio_dev->dev,
466 				     &buffer->scan_el_dev_attr_list);
467 	if (ret)
468 		return ret;
469 	attrcount++;
470 	if (chan->type != IIO_TIMESTAMP)
471 		ret = __iio_add_chan_devattr("en",
472 					     chan,
473 					     &iio_scan_el_show,
474 					     &iio_scan_el_store,
475 					     chan->scan_index,
476 					     0,
477 					     &indio_dev->dev,
478 					     &buffer->scan_el_dev_attr_list);
479 	else
480 		ret = __iio_add_chan_devattr("en",
481 					     chan,
482 					     &iio_scan_el_ts_show,
483 					     &iio_scan_el_ts_store,
484 					     chan->scan_index,
485 					     0,
486 					     &indio_dev->dev,
487 					     &buffer->scan_el_dev_attr_list);
488 	if (ret)
489 		return ret;
490 	attrcount++;
491 	ret = attrcount;
492 	return ret;
493 }
494 
495 static ssize_t iio_buffer_read_length(struct device *dev,
496 				      struct device_attribute *attr,
497 				      char *buf)
498 {
499 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
500 	struct iio_buffer *buffer = indio_dev->buffer;
501 
502 	return sprintf(buf, "%d\n", buffer->length);
503 }
504 
505 static ssize_t iio_buffer_write_length(struct device *dev,
506 				       struct device_attribute *attr,
507 				       const char *buf, size_t len)
508 {
509 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
510 	struct iio_buffer *buffer = indio_dev->buffer;
511 	unsigned int val;
512 	int ret;
513 
514 	ret = kstrtouint(buf, 10, &val);
515 	if (ret)
516 		return ret;
517 
518 	if (val == buffer->length)
519 		return len;
520 
521 	mutex_lock(&indio_dev->mlock);
522 	if (iio_buffer_is_active(indio_dev->buffer)) {
523 		ret = -EBUSY;
524 	} else {
525 		buffer->access->set_length(buffer, val);
526 		ret = 0;
527 	}
528 	if (ret)
529 		goto out;
530 	if (buffer->length && buffer->length < buffer->watermark)
531 		buffer->watermark = buffer->length;
532 out:
533 	mutex_unlock(&indio_dev->mlock);
534 
535 	return ret ? ret : len;
536 }
537 
538 static ssize_t iio_buffer_show_enable(struct device *dev,
539 				      struct device_attribute *attr,
540 				      char *buf)
541 {
542 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
543 	return sprintf(buf, "%d\n", iio_buffer_is_active(indio_dev->buffer));
544 }
545 
546 static unsigned int iio_storage_bytes_for_si(struct iio_dev *indio_dev,
547 					     unsigned int scan_index)
548 {
549 	const struct iio_chan_spec *ch;
550 	unsigned int bytes;
551 
552 	ch = iio_find_channel_from_si(indio_dev, scan_index);
553 	bytes = ch->scan_type.storagebits / 8;
554 	if (ch->scan_type.repeat > 1)
555 		bytes *= ch->scan_type.repeat;
556 	return bytes;
557 }
558 
559 static unsigned int iio_storage_bytes_for_timestamp(struct iio_dev *indio_dev)
560 {
561 	return iio_storage_bytes_for_si(indio_dev,
562 					indio_dev->scan_index_timestamp);
563 }
564 
565 static int iio_compute_scan_bytes(struct iio_dev *indio_dev,
566 				const unsigned long *mask, bool timestamp)
567 {
568 	unsigned bytes = 0;
569 	int length, i, largest = 0;
570 
571 	/* How much space will the demuxed element take? */
572 	for_each_set_bit(i, mask,
573 			 indio_dev->masklength) {
574 		length = iio_storage_bytes_for_si(indio_dev, i);
575 		bytes = ALIGN(bytes, length);
576 		bytes += length;
577 		largest = max(largest, length);
578 	}
579 
580 	if (timestamp) {
581 		length = iio_storage_bytes_for_timestamp(indio_dev);
582 		bytes = ALIGN(bytes, length);
583 		bytes += length;
584 		largest = max(largest, length);
585 	}
586 
587 	bytes = ALIGN(bytes, largest);
588 	return bytes;
589 }
590 
591 static void iio_buffer_activate(struct iio_dev *indio_dev,
592 	struct iio_buffer *buffer)
593 {
594 	iio_buffer_get(buffer);
595 	list_add(&buffer->buffer_list, &indio_dev->buffer_list);
596 }
597 
598 static void iio_buffer_deactivate(struct iio_buffer *buffer)
599 {
600 	list_del_init(&buffer->buffer_list);
601 	wake_up_interruptible(&buffer->pollq);
602 	iio_buffer_put(buffer);
603 }
604 
605 static void iio_buffer_deactivate_all(struct iio_dev *indio_dev)
606 {
607 	struct iio_buffer *buffer, *_buffer;
608 
609 	list_for_each_entry_safe(buffer, _buffer,
610 			&indio_dev->buffer_list, buffer_list)
611 		iio_buffer_deactivate(buffer);
612 }
613 
614 static int iio_buffer_enable(struct iio_buffer *buffer,
615 	struct iio_dev *indio_dev)
616 {
617 	if (!buffer->access->enable)
618 		return 0;
619 	return buffer->access->enable(buffer, indio_dev);
620 }
621 
622 static int iio_buffer_disable(struct iio_buffer *buffer,
623 	struct iio_dev *indio_dev)
624 {
625 	if (!buffer->access->disable)
626 		return 0;
627 	return buffer->access->disable(buffer, indio_dev);
628 }
629 
630 static void iio_buffer_update_bytes_per_datum(struct iio_dev *indio_dev,
631 	struct iio_buffer *buffer)
632 {
633 	unsigned int bytes;
634 
635 	if (!buffer->access->set_bytes_per_datum)
636 		return;
637 
638 	bytes = iio_compute_scan_bytes(indio_dev, buffer->scan_mask,
639 		buffer->scan_timestamp);
640 
641 	buffer->access->set_bytes_per_datum(buffer, bytes);
642 }
643 
644 static int iio_buffer_request_update(struct iio_dev *indio_dev,
645 	struct iio_buffer *buffer)
646 {
647 	int ret;
648 
649 	iio_buffer_update_bytes_per_datum(indio_dev, buffer);
650 	if (buffer->access->request_update) {
651 		ret = buffer->access->request_update(buffer);
652 		if (ret) {
653 			dev_dbg(&indio_dev->dev,
654 			       "Buffer not started: buffer parameter update failed (%d)\n",
655 				ret);
656 			return ret;
657 		}
658 	}
659 
660 	return 0;
661 }
662 
663 static void iio_free_scan_mask(struct iio_dev *indio_dev,
664 	const unsigned long *mask)
665 {
666 	/* If the mask is dynamically allocated free it, otherwise do nothing */
667 	if (!indio_dev->available_scan_masks)
668 		bitmap_free(mask);
669 }
670 
671 struct iio_device_config {
672 	unsigned int mode;
673 	unsigned int watermark;
674 	const unsigned long *scan_mask;
675 	unsigned int scan_bytes;
676 	bool scan_timestamp;
677 };
678 
679 static int iio_verify_update(struct iio_dev *indio_dev,
680 	struct iio_buffer *insert_buffer, struct iio_buffer *remove_buffer,
681 	struct iio_device_config *config)
682 {
683 	unsigned long *compound_mask;
684 	const unsigned long *scan_mask;
685 	bool strict_scanmask = false;
686 	struct iio_buffer *buffer;
687 	bool scan_timestamp;
688 	unsigned int modes;
689 
690 	memset(config, 0, sizeof(*config));
691 	config->watermark = ~0;
692 
693 	/*
694 	 * If there is just one buffer and we are removing it there is nothing
695 	 * to verify.
696 	 */
697 	if (remove_buffer && !insert_buffer &&
698 		list_is_singular(&indio_dev->buffer_list))
699 			return 0;
700 
701 	modes = indio_dev->modes;
702 
703 	list_for_each_entry(buffer, &indio_dev->buffer_list, buffer_list) {
704 		if (buffer == remove_buffer)
705 			continue;
706 		modes &= buffer->access->modes;
707 		config->watermark = min(config->watermark, buffer->watermark);
708 	}
709 
710 	if (insert_buffer) {
711 		modes &= insert_buffer->access->modes;
712 		config->watermark = min(config->watermark,
713 			insert_buffer->watermark);
714 	}
715 
716 	/* Definitely possible for devices to support both of these. */
717 	if ((modes & INDIO_BUFFER_TRIGGERED) && indio_dev->trig) {
718 		config->mode = INDIO_BUFFER_TRIGGERED;
719 	} else if (modes & INDIO_BUFFER_HARDWARE) {
720 		/*
721 		 * Keep things simple for now and only allow a single buffer to
722 		 * be connected in hardware mode.
723 		 */
724 		if (insert_buffer && !list_empty(&indio_dev->buffer_list))
725 			return -EINVAL;
726 		config->mode = INDIO_BUFFER_HARDWARE;
727 		strict_scanmask = true;
728 	} else if (modes & INDIO_BUFFER_SOFTWARE) {
729 		config->mode = INDIO_BUFFER_SOFTWARE;
730 	} else {
731 		/* Can only occur on first buffer */
732 		if (indio_dev->modes & INDIO_BUFFER_TRIGGERED)
733 			dev_dbg(&indio_dev->dev, "Buffer not started: no trigger\n");
734 		return -EINVAL;
735 	}
736 
737 	/* What scan mask do we actually have? */
738 	compound_mask = bitmap_zalloc(indio_dev->masklength, GFP_KERNEL);
739 	if (compound_mask == NULL)
740 		return -ENOMEM;
741 
742 	scan_timestamp = false;
743 
744 	list_for_each_entry(buffer, &indio_dev->buffer_list, buffer_list) {
745 		if (buffer == remove_buffer)
746 			continue;
747 		bitmap_or(compound_mask, compound_mask, buffer->scan_mask,
748 			  indio_dev->masklength);
749 		scan_timestamp |= buffer->scan_timestamp;
750 	}
751 
752 	if (insert_buffer) {
753 		bitmap_or(compound_mask, compound_mask,
754 			  insert_buffer->scan_mask, indio_dev->masklength);
755 		scan_timestamp |= insert_buffer->scan_timestamp;
756 	}
757 
758 	if (indio_dev->available_scan_masks) {
759 		scan_mask = iio_scan_mask_match(indio_dev->available_scan_masks,
760 				    indio_dev->masklength,
761 				    compound_mask,
762 				    strict_scanmask);
763 		bitmap_free(compound_mask);
764 		if (scan_mask == NULL)
765 			return -EINVAL;
766 	} else {
767 	    scan_mask = compound_mask;
768 	}
769 
770 	config->scan_bytes = iio_compute_scan_bytes(indio_dev,
771 				    scan_mask, scan_timestamp);
772 	config->scan_mask = scan_mask;
773 	config->scan_timestamp = scan_timestamp;
774 
775 	return 0;
776 }
777 
778 /**
779  * struct iio_demux_table - table describing demux memcpy ops
780  * @from:	index to copy from
781  * @to:		index to copy to
782  * @length:	how many bytes to copy
783  * @l:		list head used for management
784  */
785 struct iio_demux_table {
786 	unsigned from;
787 	unsigned to;
788 	unsigned length;
789 	struct list_head l;
790 };
791 
792 static void iio_buffer_demux_free(struct iio_buffer *buffer)
793 {
794 	struct iio_demux_table *p, *q;
795 	list_for_each_entry_safe(p, q, &buffer->demux_list, l) {
796 		list_del(&p->l);
797 		kfree(p);
798 	}
799 }
800 
801 static int iio_buffer_add_demux(struct iio_buffer *buffer,
802 	struct iio_demux_table **p, unsigned int in_loc, unsigned int out_loc,
803 	unsigned int length)
804 {
805 
806 	if (*p && (*p)->from + (*p)->length == in_loc &&
807 		(*p)->to + (*p)->length == out_loc) {
808 		(*p)->length += length;
809 	} else {
810 		*p = kmalloc(sizeof(**p), GFP_KERNEL);
811 		if (*p == NULL)
812 			return -ENOMEM;
813 		(*p)->from = in_loc;
814 		(*p)->to = out_loc;
815 		(*p)->length = length;
816 		list_add_tail(&(*p)->l, &buffer->demux_list);
817 	}
818 
819 	return 0;
820 }
821 
822 static int iio_buffer_update_demux(struct iio_dev *indio_dev,
823 				   struct iio_buffer *buffer)
824 {
825 	int ret, in_ind = -1, out_ind, length;
826 	unsigned in_loc = 0, out_loc = 0;
827 	struct iio_demux_table *p = NULL;
828 
829 	/* Clear out any old demux */
830 	iio_buffer_demux_free(buffer);
831 	kfree(buffer->demux_bounce);
832 	buffer->demux_bounce = NULL;
833 
834 	/* First work out which scan mode we will actually have */
835 	if (bitmap_equal(indio_dev->active_scan_mask,
836 			 buffer->scan_mask,
837 			 indio_dev->masklength))
838 		return 0;
839 
840 	/* Now we have the two masks, work from least sig and build up sizes */
841 	for_each_set_bit(out_ind,
842 			 buffer->scan_mask,
843 			 indio_dev->masklength) {
844 		in_ind = find_next_bit(indio_dev->active_scan_mask,
845 				       indio_dev->masklength,
846 				       in_ind + 1);
847 		while (in_ind != out_ind) {
848 			in_ind = find_next_bit(indio_dev->active_scan_mask,
849 					       indio_dev->masklength,
850 					       in_ind + 1);
851 			length = iio_storage_bytes_for_si(indio_dev, in_ind);
852 			/* Make sure we are aligned */
853 			in_loc = roundup(in_loc, length) + length;
854 		}
855 		length = iio_storage_bytes_for_si(indio_dev, in_ind);
856 		out_loc = roundup(out_loc, length);
857 		in_loc = roundup(in_loc, length);
858 		ret = iio_buffer_add_demux(buffer, &p, in_loc, out_loc, length);
859 		if (ret)
860 			goto error_clear_mux_table;
861 		out_loc += length;
862 		in_loc += length;
863 	}
864 	/* Relies on scan_timestamp being last */
865 	if (buffer->scan_timestamp) {
866 		length = iio_storage_bytes_for_timestamp(indio_dev);
867 		out_loc = roundup(out_loc, length);
868 		in_loc = roundup(in_loc, length);
869 		ret = iio_buffer_add_demux(buffer, &p, in_loc, out_loc, length);
870 		if (ret)
871 			goto error_clear_mux_table;
872 		out_loc += length;
873 		in_loc += length;
874 	}
875 	buffer->demux_bounce = kzalloc(out_loc, GFP_KERNEL);
876 	if (buffer->demux_bounce == NULL) {
877 		ret = -ENOMEM;
878 		goto error_clear_mux_table;
879 	}
880 	return 0;
881 
882 error_clear_mux_table:
883 	iio_buffer_demux_free(buffer);
884 
885 	return ret;
886 }
887 
888 static int iio_update_demux(struct iio_dev *indio_dev)
889 {
890 	struct iio_buffer *buffer;
891 	int ret;
892 
893 	list_for_each_entry(buffer, &indio_dev->buffer_list, buffer_list) {
894 		ret = iio_buffer_update_demux(indio_dev, buffer);
895 		if (ret < 0)
896 			goto error_clear_mux_table;
897 	}
898 	return 0;
899 
900 error_clear_mux_table:
901 	list_for_each_entry(buffer, &indio_dev->buffer_list, buffer_list)
902 		iio_buffer_demux_free(buffer);
903 
904 	return ret;
905 }
906 
907 static int iio_enable_buffers(struct iio_dev *indio_dev,
908 	struct iio_device_config *config)
909 {
910 	struct iio_buffer *buffer;
911 	int ret;
912 
913 	indio_dev->active_scan_mask = config->scan_mask;
914 	indio_dev->scan_timestamp = config->scan_timestamp;
915 	indio_dev->scan_bytes = config->scan_bytes;
916 
917 	iio_update_demux(indio_dev);
918 
919 	/* Wind up again */
920 	if (indio_dev->setup_ops->preenable) {
921 		ret = indio_dev->setup_ops->preenable(indio_dev);
922 		if (ret) {
923 			dev_dbg(&indio_dev->dev,
924 			       "Buffer not started: buffer preenable failed (%d)\n", ret);
925 			goto err_undo_config;
926 		}
927 	}
928 
929 	if (indio_dev->info->update_scan_mode) {
930 		ret = indio_dev->info
931 			->update_scan_mode(indio_dev,
932 					   indio_dev->active_scan_mask);
933 		if (ret < 0) {
934 			dev_dbg(&indio_dev->dev,
935 				"Buffer not started: update scan mode failed (%d)\n",
936 				ret);
937 			goto err_run_postdisable;
938 		}
939 	}
940 
941 	if (indio_dev->info->hwfifo_set_watermark)
942 		indio_dev->info->hwfifo_set_watermark(indio_dev,
943 			config->watermark);
944 
945 	list_for_each_entry(buffer, &indio_dev->buffer_list, buffer_list) {
946 		ret = iio_buffer_enable(buffer, indio_dev);
947 		if (ret)
948 			goto err_disable_buffers;
949 	}
950 
951 	indio_dev->currentmode = config->mode;
952 
953 	if (indio_dev->setup_ops->postenable) {
954 		ret = indio_dev->setup_ops->postenable(indio_dev);
955 		if (ret) {
956 			dev_dbg(&indio_dev->dev,
957 			       "Buffer not started: postenable failed (%d)\n", ret);
958 			goto err_disable_buffers;
959 		}
960 	}
961 
962 	return 0;
963 
964 err_disable_buffers:
965 	list_for_each_entry_continue_reverse(buffer, &indio_dev->buffer_list,
966 					     buffer_list)
967 		iio_buffer_disable(buffer, indio_dev);
968 err_run_postdisable:
969 	indio_dev->currentmode = INDIO_DIRECT_MODE;
970 	if (indio_dev->setup_ops->postdisable)
971 		indio_dev->setup_ops->postdisable(indio_dev);
972 err_undo_config:
973 	indio_dev->active_scan_mask = NULL;
974 
975 	return ret;
976 }
977 
978 static int iio_disable_buffers(struct iio_dev *indio_dev)
979 {
980 	struct iio_buffer *buffer;
981 	int ret = 0;
982 	int ret2;
983 
984 	/* Wind down existing buffers - iff there are any */
985 	if (list_empty(&indio_dev->buffer_list))
986 		return 0;
987 
988 	/*
989 	 * If things go wrong at some step in disable we still need to continue
990 	 * to perform the other steps, otherwise we leave the device in a
991 	 * inconsistent state. We return the error code for the first error we
992 	 * encountered.
993 	 */
994 
995 	if (indio_dev->setup_ops->predisable) {
996 		ret2 = indio_dev->setup_ops->predisable(indio_dev);
997 		if (ret2 && !ret)
998 			ret = ret2;
999 	}
1000 
1001 	list_for_each_entry(buffer, &indio_dev->buffer_list, buffer_list) {
1002 		ret2 = iio_buffer_disable(buffer, indio_dev);
1003 		if (ret2 && !ret)
1004 			ret = ret2;
1005 	}
1006 
1007 	indio_dev->currentmode = INDIO_DIRECT_MODE;
1008 
1009 	if (indio_dev->setup_ops->postdisable) {
1010 		ret2 = indio_dev->setup_ops->postdisable(indio_dev);
1011 		if (ret2 && !ret)
1012 			ret = ret2;
1013 	}
1014 
1015 	iio_free_scan_mask(indio_dev, indio_dev->active_scan_mask);
1016 	indio_dev->active_scan_mask = NULL;
1017 
1018 	return ret;
1019 }
1020 
1021 static int __iio_update_buffers(struct iio_dev *indio_dev,
1022 		       struct iio_buffer *insert_buffer,
1023 		       struct iio_buffer *remove_buffer)
1024 {
1025 	struct iio_device_config new_config;
1026 	int ret;
1027 
1028 	ret = iio_verify_update(indio_dev, insert_buffer, remove_buffer,
1029 		&new_config);
1030 	if (ret)
1031 		return ret;
1032 
1033 	if (insert_buffer) {
1034 		ret = iio_buffer_request_update(indio_dev, insert_buffer);
1035 		if (ret)
1036 			goto err_free_config;
1037 	}
1038 
1039 	ret = iio_disable_buffers(indio_dev);
1040 	if (ret)
1041 		goto err_deactivate_all;
1042 
1043 	if (remove_buffer)
1044 		iio_buffer_deactivate(remove_buffer);
1045 	if (insert_buffer)
1046 		iio_buffer_activate(indio_dev, insert_buffer);
1047 
1048 	/* If no buffers in list, we are done */
1049 	if (list_empty(&indio_dev->buffer_list))
1050 		return 0;
1051 
1052 	ret = iio_enable_buffers(indio_dev, &new_config);
1053 	if (ret)
1054 		goto err_deactivate_all;
1055 
1056 	return 0;
1057 
1058 err_deactivate_all:
1059 	/*
1060 	 * We've already verified that the config is valid earlier. If things go
1061 	 * wrong in either enable or disable the most likely reason is an IO
1062 	 * error from the device. In this case there is no good recovery
1063 	 * strategy. Just make sure to disable everything and leave the device
1064 	 * in a sane state.  With a bit of luck the device might come back to
1065 	 * life again later and userspace can try again.
1066 	 */
1067 	iio_buffer_deactivate_all(indio_dev);
1068 
1069 err_free_config:
1070 	iio_free_scan_mask(indio_dev, new_config.scan_mask);
1071 	return ret;
1072 }
1073 
1074 int iio_update_buffers(struct iio_dev *indio_dev,
1075 		       struct iio_buffer *insert_buffer,
1076 		       struct iio_buffer *remove_buffer)
1077 {
1078 	int ret;
1079 
1080 	if (insert_buffer == remove_buffer)
1081 		return 0;
1082 
1083 	mutex_lock(&indio_dev->info_exist_lock);
1084 	mutex_lock(&indio_dev->mlock);
1085 
1086 	if (insert_buffer && iio_buffer_is_active(insert_buffer))
1087 		insert_buffer = NULL;
1088 
1089 	if (remove_buffer && !iio_buffer_is_active(remove_buffer))
1090 		remove_buffer = NULL;
1091 
1092 	if (!insert_buffer && !remove_buffer) {
1093 		ret = 0;
1094 		goto out_unlock;
1095 	}
1096 
1097 	if (indio_dev->info == NULL) {
1098 		ret = -ENODEV;
1099 		goto out_unlock;
1100 	}
1101 
1102 	ret = __iio_update_buffers(indio_dev, insert_buffer, remove_buffer);
1103 
1104 out_unlock:
1105 	mutex_unlock(&indio_dev->mlock);
1106 	mutex_unlock(&indio_dev->info_exist_lock);
1107 
1108 	return ret;
1109 }
1110 EXPORT_SYMBOL_GPL(iio_update_buffers);
1111 
1112 void iio_disable_all_buffers(struct iio_dev *indio_dev)
1113 {
1114 	iio_disable_buffers(indio_dev);
1115 	iio_buffer_deactivate_all(indio_dev);
1116 }
1117 
1118 static ssize_t iio_buffer_store_enable(struct device *dev,
1119 				       struct device_attribute *attr,
1120 				       const char *buf,
1121 				       size_t len)
1122 {
1123 	int ret;
1124 	bool requested_state;
1125 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
1126 	bool inlist;
1127 
1128 	ret = strtobool(buf, &requested_state);
1129 	if (ret < 0)
1130 		return ret;
1131 
1132 	mutex_lock(&indio_dev->mlock);
1133 
1134 	/* Find out if it is in the list */
1135 	inlist = iio_buffer_is_active(indio_dev->buffer);
1136 	/* Already in desired state */
1137 	if (inlist == requested_state)
1138 		goto done;
1139 
1140 	if (requested_state)
1141 		ret = __iio_update_buffers(indio_dev,
1142 					 indio_dev->buffer, NULL);
1143 	else
1144 		ret = __iio_update_buffers(indio_dev,
1145 					 NULL, indio_dev->buffer);
1146 
1147 done:
1148 	mutex_unlock(&indio_dev->mlock);
1149 	return (ret < 0) ? ret : len;
1150 }
1151 
1152 static const char * const iio_scan_elements_group_name = "scan_elements";
1153 
1154 static ssize_t iio_buffer_show_watermark(struct device *dev,
1155 					 struct device_attribute *attr,
1156 					 char *buf)
1157 {
1158 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
1159 	struct iio_buffer *buffer = indio_dev->buffer;
1160 
1161 	return sprintf(buf, "%u\n", buffer->watermark);
1162 }
1163 
1164 static ssize_t iio_buffer_store_watermark(struct device *dev,
1165 					  struct device_attribute *attr,
1166 					  const char *buf,
1167 					  size_t len)
1168 {
1169 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
1170 	struct iio_buffer *buffer = indio_dev->buffer;
1171 	unsigned int val;
1172 	int ret;
1173 
1174 	ret = kstrtouint(buf, 10, &val);
1175 	if (ret)
1176 		return ret;
1177 	if (!val)
1178 		return -EINVAL;
1179 
1180 	mutex_lock(&indio_dev->mlock);
1181 
1182 	if (val > buffer->length) {
1183 		ret = -EINVAL;
1184 		goto out;
1185 	}
1186 
1187 	if (iio_buffer_is_active(indio_dev->buffer)) {
1188 		ret = -EBUSY;
1189 		goto out;
1190 	}
1191 
1192 	buffer->watermark = val;
1193 out:
1194 	mutex_unlock(&indio_dev->mlock);
1195 
1196 	return ret ? ret : len;
1197 }
1198 
1199 static ssize_t iio_dma_show_data_available(struct device *dev,
1200 						struct device_attribute *attr,
1201 						char *buf)
1202 {
1203 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
1204 	size_t bytes;
1205 
1206 	bytes = iio_buffer_data_available(indio_dev->buffer);
1207 
1208 	return sprintf(buf, "%zu\n", bytes);
1209 }
1210 
1211 static DEVICE_ATTR(length, S_IRUGO | S_IWUSR, iio_buffer_read_length,
1212 		   iio_buffer_write_length);
1213 static struct device_attribute dev_attr_length_ro = __ATTR(length,
1214 	S_IRUGO, iio_buffer_read_length, NULL);
1215 static DEVICE_ATTR(enable, S_IRUGO | S_IWUSR,
1216 		   iio_buffer_show_enable, iio_buffer_store_enable);
1217 static DEVICE_ATTR(watermark, S_IRUGO | S_IWUSR,
1218 		   iio_buffer_show_watermark, iio_buffer_store_watermark);
1219 static struct device_attribute dev_attr_watermark_ro = __ATTR(watermark,
1220 	S_IRUGO, iio_buffer_show_watermark, NULL);
1221 static DEVICE_ATTR(data_available, S_IRUGO,
1222 		iio_dma_show_data_available, NULL);
1223 
1224 static struct attribute *iio_buffer_attrs[] = {
1225 	&dev_attr_length.attr,
1226 	&dev_attr_enable.attr,
1227 	&dev_attr_watermark.attr,
1228 	&dev_attr_data_available.attr,
1229 };
1230 
1231 int iio_buffer_alloc_sysfs_and_mask(struct iio_dev *indio_dev)
1232 {
1233 	struct iio_dev_attr *p;
1234 	struct attribute **attr;
1235 	struct iio_buffer *buffer = indio_dev->buffer;
1236 	int ret, i, attrn, attrcount, attrcount_orig = 0;
1237 	const struct iio_chan_spec *channels;
1238 
1239 	channels = indio_dev->channels;
1240 	if (channels) {
1241 		int ml = indio_dev->masklength;
1242 
1243 		for (i = 0; i < indio_dev->num_channels; i++)
1244 			ml = max(ml, channels[i].scan_index + 1);
1245 		indio_dev->masklength = ml;
1246 	}
1247 
1248 	if (!buffer)
1249 		return 0;
1250 
1251 	attrcount = 0;
1252 	if (buffer->attrs) {
1253 		while (buffer->attrs[attrcount] != NULL)
1254 			attrcount++;
1255 	}
1256 
1257 	attr = kcalloc(attrcount + ARRAY_SIZE(iio_buffer_attrs) + 1,
1258 		       sizeof(struct attribute *), GFP_KERNEL);
1259 	if (!attr)
1260 		return -ENOMEM;
1261 
1262 	memcpy(attr, iio_buffer_attrs, sizeof(iio_buffer_attrs));
1263 	if (!buffer->access->set_length)
1264 		attr[0] = &dev_attr_length_ro.attr;
1265 
1266 	if (buffer->access->flags & INDIO_BUFFER_FLAG_FIXED_WATERMARK)
1267 		attr[2] = &dev_attr_watermark_ro.attr;
1268 
1269 	if (buffer->attrs)
1270 		memcpy(&attr[ARRAY_SIZE(iio_buffer_attrs)], buffer->attrs,
1271 		       sizeof(struct attribute *) * attrcount);
1272 
1273 	attr[attrcount + ARRAY_SIZE(iio_buffer_attrs)] = NULL;
1274 
1275 	buffer->buffer_group.name = "buffer";
1276 	buffer->buffer_group.attrs = attr;
1277 
1278 	indio_dev->groups[indio_dev->groupcounter++] = &buffer->buffer_group;
1279 
1280 	if (buffer->scan_el_attrs != NULL) {
1281 		attr = buffer->scan_el_attrs->attrs;
1282 		while (*attr++ != NULL)
1283 			attrcount_orig++;
1284 	}
1285 	attrcount = attrcount_orig;
1286 	INIT_LIST_HEAD(&buffer->scan_el_dev_attr_list);
1287 	channels = indio_dev->channels;
1288 	if (channels) {
1289 		/* new magic */
1290 		for (i = 0; i < indio_dev->num_channels; i++) {
1291 			if (channels[i].scan_index < 0)
1292 				continue;
1293 
1294 			ret = iio_buffer_add_channel_sysfs(indio_dev,
1295 							 &channels[i]);
1296 			if (ret < 0)
1297 				goto error_cleanup_dynamic;
1298 			attrcount += ret;
1299 			if (channels[i].type == IIO_TIMESTAMP)
1300 				indio_dev->scan_index_timestamp =
1301 					channels[i].scan_index;
1302 		}
1303 		if (indio_dev->masklength && buffer->scan_mask == NULL) {
1304 			buffer->scan_mask = bitmap_zalloc(indio_dev->masklength,
1305 							  GFP_KERNEL);
1306 			if (buffer->scan_mask == NULL) {
1307 				ret = -ENOMEM;
1308 				goto error_cleanup_dynamic;
1309 			}
1310 		}
1311 	}
1312 
1313 	buffer->scan_el_group.name = iio_scan_elements_group_name;
1314 
1315 	buffer->scan_el_group.attrs = kcalloc(attrcount + 1,
1316 					      sizeof(buffer->scan_el_group.attrs[0]),
1317 					      GFP_KERNEL);
1318 	if (buffer->scan_el_group.attrs == NULL) {
1319 		ret = -ENOMEM;
1320 		goto error_free_scan_mask;
1321 	}
1322 	if (buffer->scan_el_attrs)
1323 		memcpy(buffer->scan_el_group.attrs, buffer->scan_el_attrs,
1324 		       sizeof(buffer->scan_el_group.attrs[0])*attrcount_orig);
1325 	attrn = attrcount_orig;
1326 
1327 	list_for_each_entry(p, &buffer->scan_el_dev_attr_list, l)
1328 		buffer->scan_el_group.attrs[attrn++] = &p->dev_attr.attr;
1329 	indio_dev->groups[indio_dev->groupcounter++] = &buffer->scan_el_group;
1330 
1331 	return 0;
1332 
1333 error_free_scan_mask:
1334 	bitmap_free(buffer->scan_mask);
1335 error_cleanup_dynamic:
1336 	iio_free_chan_devattr_list(&buffer->scan_el_dev_attr_list);
1337 	kfree(indio_dev->buffer->buffer_group.attrs);
1338 
1339 	return ret;
1340 }
1341 
1342 void iio_buffer_free_sysfs_and_mask(struct iio_dev *indio_dev)
1343 {
1344 	if (!indio_dev->buffer)
1345 		return;
1346 
1347 	bitmap_free(indio_dev->buffer->scan_mask);
1348 	kfree(indio_dev->buffer->buffer_group.attrs);
1349 	kfree(indio_dev->buffer->scan_el_group.attrs);
1350 	iio_free_chan_devattr_list(&indio_dev->buffer->scan_el_dev_attr_list);
1351 }
1352 
1353 /**
1354  * iio_validate_scan_mask_onehot() - Validates that exactly one channel is selected
1355  * @indio_dev: the iio device
1356  * @mask: scan mask to be checked
1357  *
1358  * Return true if exactly one bit is set in the scan mask, false otherwise. It
1359  * can be used for devices where only one channel can be active for sampling at
1360  * a time.
1361  */
1362 bool iio_validate_scan_mask_onehot(struct iio_dev *indio_dev,
1363 	const unsigned long *mask)
1364 {
1365 	return bitmap_weight(mask, indio_dev->masklength) == 1;
1366 }
1367 EXPORT_SYMBOL_GPL(iio_validate_scan_mask_onehot);
1368 
1369 static const void *iio_demux(struct iio_buffer *buffer,
1370 				 const void *datain)
1371 {
1372 	struct iio_demux_table *t;
1373 
1374 	if (list_empty(&buffer->demux_list))
1375 		return datain;
1376 	list_for_each_entry(t, &buffer->demux_list, l)
1377 		memcpy(buffer->demux_bounce + t->to,
1378 		       datain + t->from, t->length);
1379 
1380 	return buffer->demux_bounce;
1381 }
1382 
1383 static int iio_push_to_buffer(struct iio_buffer *buffer, const void *data)
1384 {
1385 	const void *dataout = iio_demux(buffer, data);
1386 	int ret;
1387 
1388 	ret = buffer->access->store_to(buffer, dataout);
1389 	if (ret)
1390 		return ret;
1391 
1392 	/*
1393 	 * We can't just test for watermark to decide if we wake the poll queue
1394 	 * because read may request less samples than the watermark.
1395 	 */
1396 	wake_up_interruptible_poll(&buffer->pollq, EPOLLIN | EPOLLRDNORM);
1397 	return 0;
1398 }
1399 
1400 /**
1401  * iio_push_to_buffers() - push to a registered buffer.
1402  * @indio_dev:		iio_dev structure for device.
1403  * @data:		Full scan.
1404  */
1405 int iio_push_to_buffers(struct iio_dev *indio_dev, const void *data)
1406 {
1407 	int ret;
1408 	struct iio_buffer *buf;
1409 
1410 	list_for_each_entry(buf, &indio_dev->buffer_list, buffer_list) {
1411 		ret = iio_push_to_buffer(buf, data);
1412 		if (ret < 0)
1413 			return ret;
1414 	}
1415 
1416 	return 0;
1417 }
1418 EXPORT_SYMBOL_GPL(iio_push_to_buffers);
1419 
1420 /**
1421  * iio_buffer_release() - Free a buffer's resources
1422  * @ref: Pointer to the kref embedded in the iio_buffer struct
1423  *
1424  * This function is called when the last reference to the buffer has been
1425  * dropped. It will typically free all resources allocated by the buffer. Do not
1426  * call this function manually, always use iio_buffer_put() when done using a
1427  * buffer.
1428  */
1429 static void iio_buffer_release(struct kref *ref)
1430 {
1431 	struct iio_buffer *buffer = container_of(ref, struct iio_buffer, ref);
1432 
1433 	buffer->access->release(buffer);
1434 }
1435 
1436 /**
1437  * iio_buffer_get() - Grab a reference to the buffer
1438  * @buffer: The buffer to grab a reference for, may be NULL
1439  *
1440  * Returns the pointer to the buffer that was passed into the function.
1441  */
1442 struct iio_buffer *iio_buffer_get(struct iio_buffer *buffer)
1443 {
1444 	if (buffer)
1445 		kref_get(&buffer->ref);
1446 
1447 	return buffer;
1448 }
1449 EXPORT_SYMBOL_GPL(iio_buffer_get);
1450 
1451 /**
1452  * iio_buffer_put() - Release the reference to the buffer
1453  * @buffer: The buffer to release the reference for, may be NULL
1454  */
1455 void iio_buffer_put(struct iio_buffer *buffer)
1456 {
1457 	if (buffer)
1458 		kref_put(&buffer->ref, iio_buffer_release);
1459 }
1460 EXPORT_SYMBOL_GPL(iio_buffer_put);
1461 
1462 /**
1463  * iio_device_attach_buffer - Attach a buffer to a IIO device
1464  * @indio_dev: The device the buffer should be attached to
1465  * @buffer: The buffer to attach to the device
1466  *
1467  * This function attaches a buffer to a IIO device. The buffer stays attached to
1468  * the device until the device is freed. The function should only be called at
1469  * most once per device.
1470  */
1471 void iio_device_attach_buffer(struct iio_dev *indio_dev,
1472 			      struct iio_buffer *buffer)
1473 {
1474 	indio_dev->buffer = iio_buffer_get(buffer);
1475 }
1476 EXPORT_SYMBOL_GPL(iio_device_attach_buffer);
1477