96 	struct perf_output_handle handle; /* AUX buffer output handle */
888 	/* Use AUX buffer. No need to allocate it by ourself */  in __hw_perf_event_init()
1284 	 * AUX buffer is used when in diagnostic sampling mode.  in hw_perf_event_update()
1380 static inline unsigned long aux_sdb_index(struct aux_buffer *aux,  in aux_sdb_index()  argument
1383 	return i % aux->sfb.num_sdb;  in aux_sdb_index()
1391 static inline unsigned long aux_sdb_num_alert(struct aux_buffer *aux)  in aux_sdb_num_alert()  argument
1393 	return aux_sdb_num(aux->head, aux->alert_mark);  in aux_sdb_num_alert()
1396 static inline unsigned long aux_sdb_num_empty(struct aux_buffer *aux)  in aux_sdb_num_empty()  argument
1398 	return aux_sdb_num(aux->head, aux->empty_mark);  in aux_sdb_num_empty()
1404 static struct hws_trailer_entry *aux_sdb_trailer(struct aux_buffer *aux,  in aux_sdb_trailer()  argument
1409 	index = aux_sdb_index(aux, index);  in aux_sdb_trailer()
1410 	sdb = aux->sdb_index[index];  in aux_sdb_trailer()
1416  * disabled. Collect the full SDBs in AUX buffer which have not reached
1426 	struct aux_buffer *aux;  in aux_output_end()  local
1429 	aux = perf_get_aux(handle);  in aux_output_end()
1430 	if (!aux)  in aux_output_end()
1433 	range_scan = aux_sdb_num_alert(aux);  in aux_output_end()
1434 	for (i = 0, idx = aux->head; i < range_scan; i++, idx++) {  in aux_output_end()
1435 		te = aux_sdb_trailer(aux, idx);  in aux_output_end()
1443 	te = aux_sdb_trailer(aux, aux->alert_mark);  in aux_output_end()
1447 			    __func__, i, range_scan, aux->head);  in aux_output_end()
1460 			    struct aux_buffer *aux,  in aux_output_begin()  argument
1469 	aux->head = handle->head >> PAGE_SHIFT;  in aux_output_begin()
1475 	 * SDBs between aux->head and aux->empty_mark are already ready  in aux_output_begin()
1480 			    __func__, range, aux->head, aux->alert_mark,  in aux_output_begin()
1481 			    aux->empty_mark);  in aux_output_begin()
1482 	if (range > aux_sdb_num_empty(aux)) {  in aux_output_begin()
1483 		range_scan = range - aux_sdb_num_empty(aux);  in aux_output_begin()
1484 		idx = aux->empty_mark + 1;  in aux_output_begin()
1486 			te = aux_sdb_trailer(aux, idx);  in aux_output_begin()
1492 		aux->empty_mark = aux->head + range - 1;  in aux_output_begin()
1496 	aux->alert_mark = aux->head + range/2 - 1;  in aux_output_begin()
1497 	te = aux_sdb_trailer(aux, aux->alert_mark);  in aux_output_begin()
1501 	head = aux_sdb_index(aux, aux->head);  in aux_output_begin()
1502 	base = aux->sdbt_index[head / CPUM_SF_SDB_PER_TABLE];  in aux_output_begin()
1505 	cpuhw->lsctl.dear = virt_to_phys((void *)aux->sdb_index[head]);  in aux_output_begin()
1509 			    aux->head, aux->alert_mark, aux->empty_mark,  in aux_output_begin()
1522 static bool aux_set_alert(struct aux_buffer *aux, unsigned long alert_index,  in aux_set_alert()  argument
1528 	te = aux_sdb_trailer(aux, alert_index);  in aux_set_alert()
1551  * @aux:	The AUX buffer to set
1552  * @range:	The range of SDBs to scan started from aux->head
1555  * Set alert indicator on the SDB at index of aux->alert_mark. If this SDB is
1560  * Scan the SDBs in AUX buffer from behind aux->empty_mark. They are used
1563  * If aux->alert_mark fall in this area, just set it. Overflow count is
1566  * SDBs between aux->head and aux->empty_mark are already reset at last time.
1571 static bool aux_reset_buffer(struct aux_buffer *aux, unsigned long range,  in aux_reset_buffer()  argument
1580 			    "empty %ld\n", __func__, range, aux->head,  in aux_reset_buffer()
1581 			    aux->alert_mark, aux->empty_mark);  in aux_reset_buffer()
1582 	if (range <= aux_sdb_num_empty(aux))  in aux_reset_buffer()
1588 		return aux_set_alert(aux, aux->alert_mark, overflow);  in aux_reset_buffer()
1590 	if (aux->alert_mark <= aux->empty_mark)  in aux_reset_buffer()
1595 		if (!aux_set_alert(aux, aux->alert_mark, overflow))  in aux_reset_buffer()
1603 	range_scan = range - aux_sdb_num_empty(aux);  in aux_reset_buffer()
1604 	idx_old = idx = aux->empty_mark + 1;  in aux_reset_buffer()
1606 		te = aux_sdb_trailer(aux, idx);  in aux_reset_buffer()
1614 			if (idx == aux->alert_mark)  in aux_reset_buffer()
1624 	aux->empty_mark = aux->head + range - 1;  in aux_reset_buffer()
1628 			    idx - 1, aux->empty_mark);  in aux_reset_buffer()
1637 	struct aux_buffer *aux;  in hw_collect_aux()  local
1644 	aux = perf_get_aux(handle);  in hw_collect_aux()
1645 	if (!aux)  in hw_collect_aux()
1649 	size = aux_sdb_num_alert(aux) << PAGE_SHIFT;  in hw_collect_aux()
1654 	num_sdb = aux->sfb.num_sdb;  in hw_collect_aux()
1657 		aux = perf_aux_output_begin(handle, cpuhw->event);  in hw_collect_aux()
1659 			pr_err("The AUX buffer with %lu pages for the "  in hw_collect_aux()
1664 		if (!aux)  in hw_collect_aux()
1668 		aux->head = handle->head >> PAGE_SHIFT;  in hw_collect_aux()
1671 			aux->alert_mark = aux->head;  in hw_collect_aux()
1673 			aux->alert_mark = aux->head + range/2 - 1;  in hw_collect_aux()
1675 		if (aux_reset_buffer(aux, range, &overflow)) {  in hw_collect_aux()
1682 			pr_err("Sample data caused the AUX buffer with %lu "  in hw_collect_aux()
1683 			       "pages to overflow\n", aux->sfb.num_sdb);  in hw_collect_aux()
1686 					    aux->head, range, overflow);  in hw_collect_aux()
1688 			size = aux_sdb_num_alert(aux) << PAGE_SHIFT;  in hw_collect_aux()
1693 					    aux->head, aux->alert_mark);  in hw_collect_aux()
1699 				    "empty %ld\n", __func__, aux->head,  in hw_collect_aux()
1700 				    aux->alert_mark, aux->empty_mark);  in hw_collect_aux()
1704  * Callback when freeing AUX buffers.
1708 	struct aux_buffer *aux = data;  in aux_buffer_free()  local
1711 	if (!aux)  in aux_buffer_free()
1715 	num_sdbt = aux->sfb.num_sdbt;  in aux_buffer_free()
1717 		free_page(aux->sdbt_index[i]);  in aux_buffer_free()
1719 	kfree(aux->sdbt_index);  in aux_buffer_free()
1720 	kfree(aux->sdb_index);  in aux_buffer_free()
1721 	kfree(aux);  in aux_buffer_free()
1738  * aux_buffer_setup() - Setup AUX buffer for diagnostic mode sampling
1744  * This is the callback when setup an event using AUX buffer. Perf tool can
1746  * for basic samples, AUX buffer belongs to the event. It is scheduled with
1749  * Return the private AUX buffer structure if success or NULL if fails.
1755 	struct aux_buffer *aux;  in aux_buffer_setup()  local
1763 		pr_err("AUX buffer size (%i pages) is larger than the "  in aux_buffer_setup()
1768 		pr_err("AUX buffer size (%i pages) is less than the "  in aux_buffer_setup()
1775 	aux = kzalloc(sizeof(struct aux_buffer), GFP_KERNEL);  in aux_buffer_setup()
1776 	if (!aux)  in aux_buffer_setup()
1778 	sfb = &aux->sfb;  in aux_buffer_setup()
1782 	aux->sdbt_index = kmalloc_array(n_sdbt, sizeof(void *), GFP_KERNEL);  in aux_buffer_setup()
1783 	if (!aux->sdbt_index)  in aux_buffer_setup()
1787 	aux->sdb_index = kmalloc_array(nr_pages, sizeof(void *), GFP_KERNEL);  in aux_buffer_setup()
1788 	if (!aux->sdb_index)  in aux_buffer_setup()
1796 	aux->sdbt_index[sfb->num_sdbt++] = (unsigned long)sfb->sdbt;  in aux_buffer_setup()
1800 	 * Link the provided pages of AUX buffer to SDBT.  in aux_buffer_setup()
1808 			aux->sdbt_index[sfb->num_sdbt++] = (unsigned long)new;  in aux_buffer_setup()
1815 		aux->sdb_index[i] = (unsigned long)pages[i];  in aux_buffer_setup()
1829 	aux->empty_mark = sfb->num_sdb - 1;  in aux_buffer_setup()
1834 	return aux;  in aux_buffer_setup()
1837 	/* SDBs (AUX buffer pages) are freed by caller */  in aux_buffer_setup()
1839 		free_page(aux->sdbt_index[i]);  in aux_buffer_setup()
1840 	kfree(aux->sdb_index);  in aux_buffer_setup()
1842 	kfree(aux->sdbt_index);  in aux_buffer_setup()
1844 	kfree(aux);  in aux_buffer_setup()
1936 	struct aux_buffer *aux;  in cpumsf_pmu_add()  local
1971 		aux = perf_aux_output_begin(&cpuhw->handle, event);  in cpumsf_pmu_add()
1972 		if (!aux) {  in cpumsf_pmu_add()
1976 		err = aux_output_begin(&cpuhw->handle, aux, cpuhw);  in cpumsf_pmu_add()