80 	struct sf_buffer sfb;  member
93 	struct sf_buffer sfb;	    /* Sampling buffer */  member
170 	return !!cpuhw->sfb.sdbt;  in sf_buffer_available()
176 static void free_sampling_buffer(struct sf_buffer *sfb)  in free_sampling_buffer()  argument
180 	if (!sfb->sdbt)  in free_sampling_buffer()
183 	sdbt = sfb->sdbt;  in free_sampling_buffer()
198 			if (curr == sfb->sdbt)  in free_sampling_buffer()
212 			    (unsigned long)sfb->sdbt);  in free_sampling_buffer()
213 	memset(sfb, 0, sizeof(*sfb));  in free_sampling_buffer()
245 static int realloc_sampling_buffer(struct sf_buffer *sfb,  in realloc_sampling_buffer()  argument
251 	if (!sfb->sdbt || !sfb->tail)  in realloc_sampling_buffer()
254 	if (!is_link_entry(sfb->tail))  in realloc_sampling_buffer()
262 	tail = sfb->tail;  in realloc_sampling_buffer()
267 	if (sfb->sdbt != get_next_sdbt(tail)) {  in realloc_sampling_buffer()
271 				    (unsigned long)sfb->sdbt,  in realloc_sampling_buffer()
286 			sfb->num_sdbt++;  in realloc_sampling_buffer()
306 				sfb->num_sdbt--;  in realloc_sampling_buffer()
312 		sfb->num_sdb++;  in realloc_sampling_buffer()
318 	*tail = virt_to_phys(sfb->sdbt) + 1;  in realloc_sampling_buffer()
319 	sfb->tail = tail;  in realloc_sampling_buffer()
323 			    sfb->num_sdbt, sfb->num_sdb);  in realloc_sampling_buffer()
338 static int alloc_sampling_buffer(struct sf_buffer *sfb, unsigned long num_sdb)  in alloc_sampling_buffer()  argument
342 	if (sfb->sdbt)  in alloc_sampling_buffer()
346 	sfb->sdbt = (unsigned long *)get_zeroed_page(GFP_KERNEL);  in alloc_sampling_buffer()
347 	if (!sfb->sdbt)  in alloc_sampling_buffer()
349 	sfb->num_sdb = 0;  in alloc_sampling_buffer()
350 	sfb->num_sdbt = 1;  in alloc_sampling_buffer()
355 	sfb->tail = sfb->sdbt;  in alloc_sampling_buffer()
356 	*sfb->tail = virt_to_phys((void *)sfb->sdbt) + 1;  in alloc_sampling_buffer()
359 	rc = realloc_sampling_buffer(sfb, num_sdb, GFP_KERNEL);  in alloc_sampling_buffer()
361 		free_sampling_buffer(sfb);  in alloc_sampling_buffer()
368 			(unsigned long)sfb->sdbt, (unsigned long)*sfb->sdbt);  in alloc_sampling_buffer()
390 static unsigned long sfb_pending_allocs(struct sf_buffer *sfb,  in sfb_pending_allocs()  argument
393 	if (!sfb->sdbt)  in sfb_pending_allocs()
395 	if (SFB_ALLOC_REG(hwc) > sfb->num_sdb)  in sfb_pending_allocs()
396 		return SFB_ALLOC_REG(hwc) - sfb->num_sdb;  in sfb_pending_allocs()
400 static int sfb_has_pending_allocs(struct sf_buffer *sfb,  in sfb_has_pending_allocs()  argument
403 	return sfb_pending_allocs(sfb, hwc) > 0;  in sfb_has_pending_allocs()
422 	if (cpuhw->sfb.sdbt)  in deallocate_buffers()
423 		free_sampling_buffer(&cpuhw->sfb);  in deallocate_buffers()
482 	return alloc_sampling_buffer(&cpuhw->sfb,  in allocate_buffers()
483 				     sfb_pending_allocs(&cpuhw->sfb, hwc));  in allocate_buffers()
530 	ratio = DIV_ROUND_UP(100 * OVERFLOW_REG(hwc) * cpuhw->sfb.num_sdb,  in sfb_account_overflows()
534 	num = compute_sfb_extent(ratio, cpuhw->sfb.num_sdb);  in sfb_account_overflows()
554 static void extend_sampling_buffer(struct sf_buffer *sfb,  in extend_sampling_buffer()  argument
560 	num = sfb_pending_allocs(sfb, hwc);  in extend_sampling_buffer()
563 	num_old = sfb->num_sdb;  in extend_sampling_buffer()
575 	rc = realloc_sampling_buffer(sfb, num, GFP_ATOMIC);  in extend_sampling_buffer()
580 	if (sfb_has_pending_allocs(sfb, hwc))  in extend_sampling_buffer()
583 				    __func__, num, sfb->num_sdb - num_old,  in extend_sampling_buffer()
584 				    sfb_pending_allocs(sfb, hwc));  in extend_sampling_buffer()
1007 			extend_sampling_buffer(&cpuhw->sfb, hwc);  in cpumsf_pmu_enable()
1383 	return i % aux->sfb.num_sdb;  in aux_sdb_index()
1654 	num_sdb = aux->sfb.num_sdb;  in hw_collect_aux()
1683 			       "pages to overflow\n", aux->sfb.num_sdb);  in hw_collect_aux()
1715 	num_sdbt = aux->sfb.num_sdbt;  in aux_buffer_free()
1754 	struct sf_buffer *sfb;  in aux_buffer_setup()  local
1778 	sfb = &aux->sfb;  in aux_buffer_setup()
1792 	sfb->num_sdbt = 0;  in aux_buffer_setup()
1793 	sfb->sdbt = (unsigned long *)get_zeroed_page(GFP_KERNEL);  in aux_buffer_setup()
1794 	if (!sfb->sdbt)  in aux_buffer_setup()
1796 	aux->sdbt_index[sfb->num_sdbt++] = (unsigned long)sfb->sdbt;  in aux_buffer_setup()
1797 	tail = sfb->tail = sfb->sdbt;  in aux_buffer_setup()
1808 			aux->sdbt_index[sfb->num_sdbt++] = (unsigned long)new;  in aux_buffer_setup()
1818 	sfb->num_sdb = nr_pages;  in aux_buffer_setup()
1821 	*tail = virt_to_phys(sfb->sdbt) + 1;  in aux_buffer_setup()
1822 	sfb->tail = tail;  in aux_buffer_setup()
1829 	aux->empty_mark = sfb->num_sdb - 1;  in aux_buffer_setup()
1832 			    sfb->num_sdbt, sfb->num_sdb);  in aux_buffer_setup()
1838 	for (i = 0; i < sfb->num_sdbt; i++)  in aux_buffer_setup()
1944 	if (!SAMPL_DIAG_MODE(&event->hw) && !cpuhw->sfb.sdbt)  in cpumsf_pmu_add()
1961 		cpuhw->lsctl.tear = virt_to_phys(cpuhw->sfb.sdbt);  in cpumsf_pmu_add()
1962 		cpuhw->lsctl.dear = *(unsigned long *)cpuhw->sfb.sdbt;  in cpumsf_pmu_add()
1963 		TEAR_REG(&event->hw) = (unsigned long)cpuhw->sfb.sdbt;  in cpumsf_pmu_add()