14 static struct page *lan966x_fdma_rx_alloc_page(struct lan966x_rx *rx,  in lan966x_fdma_rx_alloc_page()  argument
19 	page = page_pool_dev_alloc_pages(rx->page_pool);  in lan966x_fdma_rx_alloc_page()
28 static void lan966x_fdma_rx_free_pages(struct lan966x_rx *rx)  in lan966x_fdma_rx_free_pages()  argument
34 			page_pool_put_full_page(rx->page_pool,  in lan966x_fdma_rx_free_pages()
35 						rx->page[i][j], false);  in lan966x_fdma_rx_free_pages()
39 static void lan966x_fdma_rx_free_page(struct lan966x_rx *rx)  in lan966x_fdma_rx_free_page()  argument
43 	page = rx->page[rx->dcb_index][rx->db_index];  in lan966x_fdma_rx_free_page()
47 	page_pool_recycle_direct(rx->page_pool, page);  in lan966x_fdma_rx_free_page()
50 static void lan966x_fdma_rx_add_dcb(struct lan966x_rx *rx,  in lan966x_fdma_rx_add_dcb()  argument
63 	dcb->info = FDMA_DCB_INFO_DATAL(PAGE_SIZE << rx->page_order);  in lan966x_fdma_rx_add_dcb()
65 	rx->last_entry->nextptr = nextptr;  in lan966x_fdma_rx_add_dcb()
66 	rx->last_entry = dcb;  in lan966x_fdma_rx_add_dcb()
69 static int lan966x_fdma_rx_alloc_page_pool(struct lan966x_rx *rx)  in lan966x_fdma_rx_alloc_page_pool()  argument
71 	struct lan966x *lan966x = rx->lan966x;  in lan966x_fdma_rx_alloc_page_pool()
73 		.order = rx->page_order,  in lan966x_fdma_rx_alloc_page_pool()
80 		.max_len = rx->max_mtu -  in lan966x_fdma_rx_alloc_page_pool()
87 	rx->page_pool = page_pool_create(&pp_params);  in lan966x_fdma_rx_alloc_page_pool()
98 					   rx->page_pool);  in lan966x_fdma_rx_alloc_page_pool()
101 	return PTR_ERR_OR_ZERO(rx->page_pool);  in lan966x_fdma_rx_alloc_page_pool()
104 static int lan966x_fdma_rx_alloc(struct lan966x_rx *rx)  in lan966x_fdma_rx_alloc()  argument
106 	struct lan966x *lan966x = rx->lan966x;  in lan966x_fdma_rx_alloc()
113 	if (lan966x_fdma_rx_alloc_page_pool(rx))  in lan966x_fdma_rx_alloc()
114 		return PTR_ERR(rx->page_pool);  in lan966x_fdma_rx_alloc()
120 	rx->dcbs = dma_alloc_coherent(lan966x->dev, size, &rx->dma, GFP_KERNEL);  in lan966x_fdma_rx_alloc()
121 	if (!rx->dcbs)  in lan966x_fdma_rx_alloc()
124 	rx->last_entry = rx->dcbs;  in lan966x_fdma_rx_alloc()
125 	rx->db_index = 0;  in lan966x_fdma_rx_alloc()
126 	rx->dcb_index = 0;  in lan966x_fdma_rx_alloc()
130 		dcb = &rx->dcbs[i];  in lan966x_fdma_rx_alloc()
136 			page = lan966x_fdma_rx_alloc_page(rx, db);  in lan966x_fdma_rx_alloc()
141 			rx->page[i][j] = page;  in lan966x_fdma_rx_alloc()
144 		lan966x_fdma_rx_add_dcb(rx, dcb, rx->dma + sizeof(*dcb) * i);  in lan966x_fdma_rx_alloc()
150 static void lan966x_fdma_rx_advance_dcb(struct lan966x_rx *rx)  in lan966x_fdma_rx_advance_dcb()  argument
152 	rx->dcb_index++;  in lan966x_fdma_rx_advance_dcb()
153 	rx->dcb_index &= FDMA_DCB_MAX - 1;  in lan966x_fdma_rx_advance_dcb()
156 static void lan966x_fdma_rx_free(struct lan966x_rx *rx)  in lan966x_fdma_rx_free()  argument
158 	struct lan966x *lan966x = rx->lan966x;  in lan966x_fdma_rx_free()
164 	dma_free_coherent(lan966x->dev, size, rx->dcbs, rx->dma);  in lan966x_fdma_rx_free()
167 static void lan966x_fdma_rx_start(struct lan966x_rx *rx)  in lan966x_fdma_rx_start()  argument
169 	struct lan966x *lan966x = rx->lan966x;  in lan966x_fdma_rx_start()
175 	lan_wr(lower_32_bits((u64)rx->dma), lan966x,  in lan966x_fdma_rx_start()
176 	       FDMA_DCB_LLP(rx->channel_id));  in lan966x_fdma_rx_start()
177 	lan_wr(upper_32_bits((u64)rx->dma), lan966x,  in lan966x_fdma_rx_start()
178 	       FDMA_DCB_LLP1(rx->channel_id));  in lan966x_fdma_rx_start()
184 	       lan966x, FDMA_CH_CFG(rx->channel_id));  in lan966x_fdma_rx_start()
194 	mask |= BIT(rx->channel_id);  in lan966x_fdma_rx_start()
200 	lan_rmw(FDMA_CH_ACTIVATE_CH_ACTIVATE_SET(BIT(rx->channel_id)),  in lan966x_fdma_rx_start()
205 static void lan966x_fdma_rx_disable(struct lan966x_rx *rx)  in lan966x_fdma_rx_disable()  argument
207 	struct lan966x *lan966x = rx->lan966x;  in lan966x_fdma_rx_disable()
211 	lan_rmw(FDMA_CH_DISABLE_CH_DISABLE_SET(BIT(rx->channel_id)),  in lan966x_fdma_rx_disable()
216 				  val, !(val & BIT(rx->channel_id)),  in lan966x_fdma_rx_disable()
219 	lan_rmw(FDMA_CH_DB_DISCARD_DB_DISCARD_SET(BIT(rx->channel_id)),  in lan966x_fdma_rx_disable()
224 static void lan966x_fdma_rx_reload(struct lan966x_rx *rx)  in lan966x_fdma_rx_reload()  argument
226 	struct lan966x *lan966x = rx->lan966x;  in lan966x_fdma_rx_reload()
228 	lan_rmw(FDMA_CH_RELOAD_CH_RELOAD_SET(BIT(rx->channel_id)),  in lan966x_fdma_rx_reload()
394 	struct lan966x_rx *rx = &lan966x->rx;  in lan966x_fdma_tx_clear_buf()  local
437 				page_pool_recycle_direct(rx->page_pool,  in lan966x_fdma_tx_clear_buf()
452 static bool lan966x_fdma_rx_more_frames(struct lan966x_rx *rx)  in lan966x_fdma_rx_more_frames()  argument
457 	db = &rx->dcbs[rx->dcb_index].db[rx->db_index];  in lan966x_fdma_rx_more_frames()
464 static int lan966x_fdma_rx_check_frame(struct lan966x_rx *rx, u64 *src_port)  in lan966x_fdma_rx_check_frame()  argument
466 	struct lan966x *lan966x = rx->lan966x;  in lan966x_fdma_rx_check_frame()
471 	db = &rx->dcbs[rx->dcb_index].db[rx->db_index];  in lan966x_fdma_rx_check_frame()
472 	page = rx->page[rx->dcb_index][rx->db_index];  in lan966x_fdma_rx_check_frame()
493 static struct sk_buff *lan966x_fdma_rx_get_frame(struct lan966x_rx *rx,  in lan966x_fdma_rx_get_frame()  argument
496 	struct lan966x *lan966x = rx->lan966x;  in lan966x_fdma_rx_get_frame()
503 	db = &rx->dcbs[rx->dcb_index].db[rx->db_index];  in lan966x_fdma_rx_get_frame()
504 	page = rx->page[rx->dcb_index][rx->db_index];  in lan966x_fdma_rx_get_frame()
506 	skb = build_skb(page_address(page), PAGE_SIZE << rx->page_order);  in lan966x_fdma_rx_get_frame()
540 	page_pool_recycle_direct(rx->page_pool, page);  in lan966x_fdma_rx_get_frame()
548 	struct lan966x_rx *rx = &lan966x->rx;  in lan966x_fdma_napi_poll()  local
549 	int dcb_reload = rx->dcb_index;  in lan966x_fdma_napi_poll()
563 		if (!lan966x_fdma_rx_more_frames(rx))  in lan966x_fdma_napi_poll()
568 		switch (lan966x_fdma_rx_check_frame(rx, &src_port)) {  in lan966x_fdma_napi_poll()
572 			lan966x_fdma_rx_free_page(rx);  in lan966x_fdma_napi_poll()
573 			lan966x_fdma_rx_advance_dcb(rx);  in lan966x_fdma_napi_poll()
579 			lan966x_fdma_rx_advance_dcb(rx);  in lan966x_fdma_napi_poll()
582 			lan966x_fdma_rx_free_page(rx);  in lan966x_fdma_napi_poll()
583 			lan966x_fdma_rx_advance_dcb(rx);  in lan966x_fdma_napi_poll()
587 		skb = lan966x_fdma_rx_get_frame(rx, src_port);  in lan966x_fdma_napi_poll()
588 		lan966x_fdma_rx_advance_dcb(rx);  in lan966x_fdma_napi_poll()
597 	while (dcb_reload != rx->dcb_index) {  in lan966x_fdma_napi_poll()
598 		db = &rx->dcbs[dcb_reload].db[rx->db_index];  in lan966x_fdma_napi_poll()
599 		page = lan966x_fdma_rx_alloc_page(rx, db);  in lan966x_fdma_napi_poll()
602 		rx->page[dcb_reload][rx->db_index] = page;  in lan966x_fdma_napi_poll()
604 		old_dcb = &rx->dcbs[dcb_reload];  in lan966x_fdma_napi_poll()
608 		nextptr = rx->dma + ((unsigned long)old_dcb -  in lan966x_fdma_napi_poll()
609 				     (unsigned long)rx->dcbs);  in lan966x_fdma_napi_poll()
610 		lan966x_fdma_rx_add_dcb(rx, old_dcb, nextptr);  in lan966x_fdma_napi_poll()
611 		lan966x_fdma_rx_reload(rx);  in lan966x_fdma_napi_poll()
917 	rx_dma = lan966x->rx.dma;  in lan966x_fdma_reload()
918 	rx_dcbs = lan966x->rx.dcbs;  in lan966x_fdma_reload()
919 	page_pool = lan966x->rx.page_pool;  in lan966x_fdma_reload()
925 	lan966x_fdma_rx_disable(&lan966x->rx);  in lan966x_fdma_reload()
926 	lan966x_fdma_rx_free_pages(&lan966x->rx);  in lan966x_fdma_reload()
927 	lan966x->rx.page_order = round_up(new_mtu, PAGE_SIZE) / PAGE_SIZE - 1;  in lan966x_fdma_reload()
928 	lan966x->rx.max_mtu = new_mtu;  in lan966x_fdma_reload()
929 	err = lan966x_fdma_rx_alloc(&lan966x->rx);  in lan966x_fdma_reload()
932 	lan966x_fdma_rx_start(&lan966x->rx);  in lan966x_fdma_reload()
945 	lan966x->rx.page_pool = page_pool;  in lan966x_fdma_reload()
946 	lan966x->rx.dma = rx_dma;  in lan966x_fdma_reload()
947 	lan966x->rx.dcbs = rx_dcbs;  in lan966x_fdma_reload()
948 	lan966x_fdma_rx_start(&lan966x->rx);  in lan966x_fdma_reload()
997 	if (max_mtu == lan966x->rx.max_mtu)  in lan966x_fdma_change_mtu()
1036 	lan966x->rx.lan966x = lan966x;  in lan966x_fdma_init()
1037 	lan966x->rx.channel_id = FDMA_XTR_CHANNEL;  in lan966x_fdma_init()
1038 	lan966x->rx.max_mtu = lan966x_fdma_get_max_frame(lan966x);  in lan966x_fdma_init()
1043 	err = lan966x_fdma_rx_alloc(&lan966x->rx);  in lan966x_fdma_init()
1049 		lan966x_fdma_rx_free(&lan966x->rx);  in lan966x_fdma_init()
1053 	lan966x_fdma_rx_start(&lan966x->rx);  in lan966x_fdma_init()
1063 	lan966x_fdma_rx_disable(&lan966x->rx);  in lan966x_fdma_deinit()
1069 	lan966x_fdma_rx_free_pages(&lan966x->rx);  in lan966x_fdma_deinit()
1070 	lan966x_fdma_rx_free(&lan966x->rx);  in lan966x_fdma_deinit()
1071 	page_pool_destroy(lan966x->rx.page_pool);  in lan966x_fdma_deinit()