168cf027fSGrygorii Strashko // SPDX-License-Identifier: GPL-2.0 284640e27SKaricheri, Muralidharan /* 384640e27SKaricheri, Muralidharan * Keystone NetCP Core driver 484640e27SKaricheri, Muralidharan * 584640e27SKaricheri, Muralidharan * Copyright (C) 2014 Texas Instruments Incorporated 684640e27SKaricheri, Muralidharan * Authors: Sandeep Nair <sandeep_n@ti.com> 784640e27SKaricheri, Muralidharan * Sandeep Paulraj <s-paulraj@ti.com> 884640e27SKaricheri, Muralidharan * Cyril Chemparathy <cyril@ti.com> 984640e27SKaricheri, Muralidharan * Santosh Shilimkar <santosh.shilimkar@ti.com> 1084640e27SKaricheri, Muralidharan * Murali Karicheri <m-karicheri2@ti.com> 1184640e27SKaricheri, Muralidharan * Wingman Kwok <w-kwok2@ti.com> 1284640e27SKaricheri, Muralidharan */ 1384640e27SKaricheri, Muralidharan 1484640e27SKaricheri, Muralidharan #include <linux/io.h> 1584640e27SKaricheri, Muralidharan #include <linux/module.h> 1684640e27SKaricheri, Muralidharan #include <linux/of_net.h> 1784640e27SKaricheri, Muralidharan #include <linux/of_address.h> 1884640e27SKaricheri, Muralidharan #include <linux/if_vlan.h> 1984640e27SKaricheri, Muralidharan #include <linux/pm_runtime.h> 2084640e27SKaricheri, Muralidharan #include <linux/platform_device.h> 2184640e27SKaricheri, Muralidharan #include <linux/soc/ti/knav_qmss.h> 2284640e27SKaricheri, Muralidharan #include <linux/soc/ti/knav_dma.h> 2384640e27SKaricheri, Muralidharan 2484640e27SKaricheri, Muralidharan #include "netcp.h" 2584640e27SKaricheri, Muralidharan 2684640e27SKaricheri, Muralidharan #define NETCP_SOP_OFFSET (NET_IP_ALIGN + NET_SKB_PAD) 2784640e27SKaricheri, Muralidharan #define NETCP_NAPI_WEIGHT 64 2884640e27SKaricheri, Muralidharan #define NETCP_TX_TIMEOUT (5 * HZ) 29866b8b18SWingMan Kwok #define NETCP_PACKET_SIZE (ETH_FRAME_LEN + ETH_FCS_LEN) 3084640e27SKaricheri, Muralidharan #define NETCP_MIN_PACKET_SIZE ETH_ZLEN 3184640e27SKaricheri, Muralidharan #define NETCP_MAX_MCAST_ADDR 16 3284640e27SKaricheri, Muralidharan 3384640e27SKaricheri, Muralidharan #define NETCP_EFUSE_REG_INDEX 0 3484640e27SKaricheri, Muralidharan 3584640e27SKaricheri, Muralidharan #define NETCP_MOD_PROBE_SKIPPED 1 3684640e27SKaricheri, Muralidharan #define NETCP_MOD_PROBE_FAILED 2 3784640e27SKaricheri, Muralidharan 3884640e27SKaricheri, Muralidharan #define NETCP_DEBUG (NETIF_MSG_HW | NETIF_MSG_WOL | \ 3984640e27SKaricheri, Muralidharan NETIF_MSG_DRV | NETIF_MSG_LINK | \ 4084640e27SKaricheri, Muralidharan NETIF_MSG_IFUP | NETIF_MSG_INTR | \ 4184640e27SKaricheri, Muralidharan NETIF_MSG_PROBE | NETIF_MSG_TIMER | \ 4284640e27SKaricheri, Muralidharan NETIF_MSG_IFDOWN | NETIF_MSG_RX_ERR | \ 4384640e27SKaricheri, Muralidharan NETIF_MSG_TX_ERR | NETIF_MSG_TX_DONE | \ 4484640e27SKaricheri, Muralidharan NETIF_MSG_PKTDATA | NETIF_MSG_TX_QUEUED | \ 4584640e27SKaricheri, Muralidharan NETIF_MSG_RX_STATUS) 4684640e27SKaricheri, Muralidharan 4771382bc0SWingMan Kwok #define NETCP_EFUSE_ADDR_SWAP 2 4871382bc0SWingMan Kwok 4984640e27SKaricheri, Muralidharan #define knav_queue_get_id(q) knav_queue_device_control(q, \ 5084640e27SKaricheri, Muralidharan KNAV_QUEUE_GET_ID, (unsigned long)NULL) 5184640e27SKaricheri, Muralidharan 5284640e27SKaricheri, Muralidharan #define knav_queue_enable_notify(q) knav_queue_device_control(q, \ 5384640e27SKaricheri, Muralidharan KNAV_QUEUE_ENABLE_NOTIFY, \ 5484640e27SKaricheri, Muralidharan (unsigned long)NULL) 5584640e27SKaricheri, Muralidharan 5684640e27SKaricheri, Muralidharan #define knav_queue_disable_notify(q) knav_queue_device_control(q, \ 5784640e27SKaricheri, Muralidharan KNAV_QUEUE_DISABLE_NOTIFY, \ 5884640e27SKaricheri, Muralidharan (unsigned long)NULL) 5984640e27SKaricheri, Muralidharan 6084640e27SKaricheri, Muralidharan #define knav_queue_get_count(q) knav_queue_device_control(q, \ 6184640e27SKaricheri, Muralidharan KNAV_QUEUE_GET_COUNT, (unsigned long)NULL) 6284640e27SKaricheri, Muralidharan 6384640e27SKaricheri, Muralidharan #define for_each_netcp_module(module) \ 6484640e27SKaricheri, Muralidharan list_for_each_entry(module, &netcp_modules, module_list) 6584640e27SKaricheri, Muralidharan 6684640e27SKaricheri, Muralidharan #define for_each_netcp_device_module(netcp_device, inst_modpriv) \ 6784640e27SKaricheri, Muralidharan list_for_each_entry(inst_modpriv, \ 6884640e27SKaricheri, Muralidharan &((netcp_device)->modpriv_head), inst_list) 6984640e27SKaricheri, Muralidharan 7084640e27SKaricheri, Muralidharan #define for_each_module(netcp, intf_modpriv) \ 7184640e27SKaricheri, Muralidharan list_for_each_entry(intf_modpriv, &netcp->module_head, intf_list) 7284640e27SKaricheri, Muralidharan 7384640e27SKaricheri, Muralidharan /* Module management structures */ 7484640e27SKaricheri, Muralidharan struct netcp_device { 7584640e27SKaricheri, Muralidharan struct list_head device_list; 7684640e27SKaricheri, Muralidharan struct list_head interface_head; 7784640e27SKaricheri, Muralidharan struct list_head modpriv_head; 7884640e27SKaricheri, Muralidharan struct device *device; 7984640e27SKaricheri, Muralidharan }; 8084640e27SKaricheri, Muralidharan 8184640e27SKaricheri, Muralidharan struct netcp_inst_modpriv { 8284640e27SKaricheri, Muralidharan struct netcp_device *netcp_device; 8384640e27SKaricheri, Muralidharan struct netcp_module *netcp_module; 8484640e27SKaricheri, Muralidharan struct list_head inst_list; 8584640e27SKaricheri, Muralidharan void *module_priv; 8684640e27SKaricheri, Muralidharan }; 8784640e27SKaricheri, Muralidharan 8884640e27SKaricheri, Muralidharan struct netcp_intf_modpriv { 8984640e27SKaricheri, Muralidharan struct netcp_intf *netcp_priv; 9084640e27SKaricheri, Muralidharan struct netcp_module *netcp_module; 9184640e27SKaricheri, Muralidharan struct list_head intf_list; 9284640e27SKaricheri, Muralidharan void *module_priv; 9384640e27SKaricheri, Muralidharan }; 9484640e27SKaricheri, Muralidharan 956246168bSWingMan Kwok struct netcp_tx_cb { 966246168bSWingMan Kwok void *ts_context; 976246168bSWingMan Kwok void (*txtstamp)(void *context, struct sk_buff *skb); 986246168bSWingMan Kwok }; 996246168bSWingMan Kwok 10084640e27SKaricheri, Muralidharan static LIST_HEAD(netcp_devices); 10184640e27SKaricheri, Muralidharan static LIST_HEAD(netcp_modules); 10284640e27SKaricheri, Muralidharan static DEFINE_MUTEX(netcp_modules_lock); 10384640e27SKaricheri, Muralidharan 10484640e27SKaricheri, Muralidharan static int netcp_debug_level = -1; 10584640e27SKaricheri, Muralidharan module_param(netcp_debug_level, int, 0); 10684640e27SKaricheri, Muralidharan MODULE_PARM_DESC(netcp_debug_level, "Netcp debug level (NETIF_MSG bits) (0=none,...,16=all)"); 10784640e27SKaricheri, Muralidharan 10884640e27SKaricheri, Muralidharan /* Helper functions - Get/Set */ 10989907779SArnd Bergmann static void get_pkt_info(dma_addr_t *buff, u32 *buff_len, dma_addr_t *ndesc, 11084640e27SKaricheri, Muralidharan struct knav_dma_desc *desc) 11184640e27SKaricheri, Muralidharan { 11289907779SArnd Bergmann *buff_len = le32_to_cpu(desc->buff_len); 11389907779SArnd Bergmann *buff = le32_to_cpu(desc->buff); 11489907779SArnd Bergmann *ndesc = le32_to_cpu(desc->next_desc); 11584640e27SKaricheri, Muralidharan } 11684640e27SKaricheri, Muralidharan 11769d707d0SKaricheri, Muralidharan static void get_desc_info(u32 *desc_info, u32 *pkt_info, 11869d707d0SKaricheri, Muralidharan struct knav_dma_desc *desc) 11969d707d0SKaricheri, Muralidharan { 12069d707d0SKaricheri, Muralidharan *desc_info = le32_to_cpu(desc->desc_info); 12169d707d0SKaricheri, Muralidharan *pkt_info = le32_to_cpu(desc->packet_info); 12269d707d0SKaricheri, Muralidharan } 12369d707d0SKaricheri, Muralidharan 12406324481SKaricheri, Muralidharan static u32 get_sw_data(int index, struct knav_dma_desc *desc) 12584640e27SKaricheri, Muralidharan { 126b1cb86aeSKaricheri, Muralidharan /* No Endian conversion needed as this data is untouched by hw */ 12706324481SKaricheri, Muralidharan return desc->sw_data[index]; 12884640e27SKaricheri, Muralidharan } 12984640e27SKaricheri, Muralidharan 13006324481SKaricheri, Muralidharan /* use these macros to get sw data */ 13106324481SKaricheri, Muralidharan #define GET_SW_DATA0(desc) get_sw_data(0, desc) 13206324481SKaricheri, Muralidharan #define GET_SW_DATA1(desc) get_sw_data(1, desc) 13306324481SKaricheri, Muralidharan #define GET_SW_DATA2(desc) get_sw_data(2, desc) 13406324481SKaricheri, Muralidharan #define GET_SW_DATA3(desc) get_sw_data(3, desc) 13589907779SArnd Bergmann 13689907779SArnd Bergmann static void get_org_pkt_info(dma_addr_t *buff, u32 *buff_len, 13784640e27SKaricheri, Muralidharan struct knav_dma_desc *desc) 13884640e27SKaricheri, Muralidharan { 13989907779SArnd Bergmann *buff = le32_to_cpu(desc->orig_buff); 14089907779SArnd Bergmann *buff_len = le32_to_cpu(desc->orig_len); 14184640e27SKaricheri, Muralidharan } 14284640e27SKaricheri, Muralidharan 14389907779SArnd Bergmann static void get_words(dma_addr_t *words, int num_words, __le32 *desc) 14484640e27SKaricheri, Muralidharan { 14584640e27SKaricheri, Muralidharan int i; 14684640e27SKaricheri, Muralidharan 14784640e27SKaricheri, Muralidharan for (i = 0; i < num_words; i++) 14889907779SArnd Bergmann words[i] = le32_to_cpu(desc[i]); 14984640e27SKaricheri, Muralidharan } 15084640e27SKaricheri, Muralidharan 15189907779SArnd Bergmann static void set_pkt_info(dma_addr_t buff, u32 buff_len, u32 ndesc, 15284640e27SKaricheri, Muralidharan struct knav_dma_desc *desc) 15384640e27SKaricheri, Muralidharan { 15489907779SArnd Bergmann desc->buff_len = cpu_to_le32(buff_len); 15589907779SArnd Bergmann desc->buff = cpu_to_le32(buff); 15689907779SArnd Bergmann desc->next_desc = cpu_to_le32(ndesc); 15784640e27SKaricheri, Muralidharan } 15884640e27SKaricheri, Muralidharan 15984640e27SKaricheri, Muralidharan static void set_desc_info(u32 desc_info, u32 pkt_info, 16084640e27SKaricheri, Muralidharan struct knav_dma_desc *desc) 16184640e27SKaricheri, Muralidharan { 16289907779SArnd Bergmann desc->desc_info = cpu_to_le32(desc_info); 16389907779SArnd Bergmann desc->packet_info = cpu_to_le32(pkt_info); 16484640e27SKaricheri, Muralidharan } 16584640e27SKaricheri, Muralidharan 16606324481SKaricheri, Muralidharan static void set_sw_data(int index, u32 data, struct knav_dma_desc *desc) 16784640e27SKaricheri, Muralidharan { 168b1cb86aeSKaricheri, Muralidharan /* No Endian conversion needed as this data is untouched by hw */ 16906324481SKaricheri, Muralidharan desc->sw_data[index] = data; 17084640e27SKaricheri, Muralidharan } 17184640e27SKaricheri, Muralidharan 17206324481SKaricheri, Muralidharan /* use these macros to set sw data */ 17306324481SKaricheri, Muralidharan #define SET_SW_DATA0(data, desc) set_sw_data(0, data, desc) 17406324481SKaricheri, Muralidharan #define SET_SW_DATA1(data, desc) set_sw_data(1, data, desc) 17506324481SKaricheri, Muralidharan #define SET_SW_DATA2(data, desc) set_sw_data(2, data, desc) 17606324481SKaricheri, Muralidharan #define SET_SW_DATA3(data, desc) set_sw_data(3, data, desc) 17706324481SKaricheri, Muralidharan 17889907779SArnd Bergmann static void set_org_pkt_info(dma_addr_t buff, u32 buff_len, 17984640e27SKaricheri, Muralidharan struct knav_dma_desc *desc) 18084640e27SKaricheri, Muralidharan { 18189907779SArnd Bergmann desc->orig_buff = cpu_to_le32(buff); 18289907779SArnd Bergmann desc->orig_len = cpu_to_le32(buff_len); 18384640e27SKaricheri, Muralidharan } 18484640e27SKaricheri, Muralidharan 18589907779SArnd Bergmann static void set_words(u32 *words, int num_words, __le32 *desc) 18684640e27SKaricheri, Muralidharan { 18784640e27SKaricheri, Muralidharan int i; 18884640e27SKaricheri, Muralidharan 18984640e27SKaricheri, Muralidharan for (i = 0; i < num_words; i++) 19089907779SArnd Bergmann desc[i] = cpu_to_le32(words[i]); 19184640e27SKaricheri, Muralidharan } 19284640e27SKaricheri, Muralidharan 19384640e27SKaricheri, Muralidharan /* Read the e-fuse value as 32 bit values to be endian independent */ 19471382bc0SWingMan Kwok static int emac_arch_get_mac_addr(char *x, void __iomem *efuse_mac, u32 swap) 19584640e27SKaricheri, Muralidharan { 19684640e27SKaricheri, Muralidharan unsigned int addr0, addr1; 19784640e27SKaricheri, Muralidharan 19884640e27SKaricheri, Muralidharan addr1 = readl(efuse_mac + 4); 19984640e27SKaricheri, Muralidharan addr0 = readl(efuse_mac); 20084640e27SKaricheri, Muralidharan 20171382bc0SWingMan Kwok switch (swap) { 20271382bc0SWingMan Kwok case NETCP_EFUSE_ADDR_SWAP: 20371382bc0SWingMan Kwok addr0 = addr1; 20471382bc0SWingMan Kwok addr1 = readl(efuse_mac); 20571382bc0SWingMan Kwok break; 20671382bc0SWingMan Kwok default: 20771382bc0SWingMan Kwok break; 20871382bc0SWingMan Kwok } 20971382bc0SWingMan Kwok 21084640e27SKaricheri, Muralidharan x[0] = (addr1 & 0x0000ff00) >> 8; 21184640e27SKaricheri, Muralidharan x[1] = addr1 & 0x000000ff; 21284640e27SKaricheri, Muralidharan x[2] = (addr0 & 0xff000000) >> 24; 21384640e27SKaricheri, Muralidharan x[3] = (addr0 & 0x00ff0000) >> 16; 21484640e27SKaricheri, Muralidharan x[4] = (addr0 & 0x0000ff00) >> 8; 21584640e27SKaricheri, Muralidharan x[5] = addr0 & 0x000000ff; 21684640e27SKaricheri, Muralidharan 21784640e27SKaricheri, Muralidharan return 0; 21884640e27SKaricheri, Muralidharan } 21984640e27SKaricheri, Muralidharan 22084640e27SKaricheri, Muralidharan /* Module management routines */ 22184640e27SKaricheri, Muralidharan static int netcp_register_interface(struct netcp_intf *netcp) 22284640e27SKaricheri, Muralidharan { 22384640e27SKaricheri, Muralidharan int ret; 22484640e27SKaricheri, Muralidharan 22584640e27SKaricheri, Muralidharan ret = register_netdev(netcp->ndev); 22684640e27SKaricheri, Muralidharan if (!ret) 22784640e27SKaricheri, Muralidharan netcp->netdev_registered = true; 22884640e27SKaricheri, Muralidharan return ret; 22984640e27SKaricheri, Muralidharan } 23084640e27SKaricheri, Muralidharan 23184640e27SKaricheri, Muralidharan static int netcp_module_probe(struct netcp_device *netcp_device, 23284640e27SKaricheri, Muralidharan struct netcp_module *module) 23384640e27SKaricheri, Muralidharan { 23484640e27SKaricheri, Muralidharan struct device *dev = netcp_device->device; 23584640e27SKaricheri, Muralidharan struct device_node *devices, *interface, *node = dev->of_node; 23684640e27SKaricheri, Muralidharan struct device_node *child; 23784640e27SKaricheri, Muralidharan struct netcp_inst_modpriv *inst_modpriv; 23884640e27SKaricheri, Muralidharan struct netcp_intf *netcp_intf; 23984640e27SKaricheri, Muralidharan struct netcp_module *tmp; 24084640e27SKaricheri, Muralidharan bool primary_module_registered = false; 24184640e27SKaricheri, Muralidharan int ret; 24284640e27SKaricheri, Muralidharan 24384640e27SKaricheri, Muralidharan /* Find this module in the sub-tree for this device */ 24484640e27SKaricheri, Muralidharan devices = of_get_child_by_name(node, "netcp-devices"); 24584640e27SKaricheri, Muralidharan if (!devices) { 24684640e27SKaricheri, Muralidharan dev_err(dev, "could not find netcp-devices node\n"); 24784640e27SKaricheri, Muralidharan return NETCP_MOD_PROBE_SKIPPED; 24884640e27SKaricheri, Muralidharan } 24984640e27SKaricheri, Muralidharan 25084640e27SKaricheri, Muralidharan for_each_available_child_of_node(devices, child) { 25121c328dcSRob Herring const char *name; 25221c328dcSRob Herring char node_name[32]; 25384640e27SKaricheri, Muralidharan 2541f43f400SMurali Karicheri if (of_property_read_string(child, "label", &name) < 0) { 25521c328dcSRob Herring snprintf(node_name, sizeof(node_name), "%pOFn", child); 25621c328dcSRob Herring name = node_name; 25721c328dcSRob Herring } 25884640e27SKaricheri, Muralidharan if (!strcasecmp(module->name, name)) 25984640e27SKaricheri, Muralidharan break; 26084640e27SKaricheri, Muralidharan } 26184640e27SKaricheri, Muralidharan 26284640e27SKaricheri, Muralidharan of_node_put(devices); 26384640e27SKaricheri, Muralidharan /* If module not used for this device, skip it */ 26484640e27SKaricheri, Muralidharan if (!child) { 26584640e27SKaricheri, Muralidharan dev_warn(dev, "module(%s) not used for device\n", module->name); 26684640e27SKaricheri, Muralidharan return NETCP_MOD_PROBE_SKIPPED; 26784640e27SKaricheri, Muralidharan } 26884640e27SKaricheri, Muralidharan 26984640e27SKaricheri, Muralidharan inst_modpriv = devm_kzalloc(dev, sizeof(*inst_modpriv), GFP_KERNEL); 27084640e27SKaricheri, Muralidharan if (!inst_modpriv) { 27184640e27SKaricheri, Muralidharan of_node_put(child); 27284640e27SKaricheri, Muralidharan return -ENOMEM; 27384640e27SKaricheri, Muralidharan } 27484640e27SKaricheri, Muralidharan 27584640e27SKaricheri, Muralidharan inst_modpriv->netcp_device = netcp_device; 27684640e27SKaricheri, Muralidharan inst_modpriv->netcp_module = module; 27784640e27SKaricheri, Muralidharan list_add_tail(&inst_modpriv->inst_list, &netcp_device->modpriv_head); 27884640e27SKaricheri, Muralidharan 27984640e27SKaricheri, Muralidharan ret = module->probe(netcp_device, dev, child, 28084640e27SKaricheri, Muralidharan &inst_modpriv->module_priv); 28184640e27SKaricheri, Muralidharan of_node_put(child); 28284640e27SKaricheri, Muralidharan if (ret) { 28384640e27SKaricheri, Muralidharan dev_err(dev, "Probe of module(%s) failed with %d\n", 28484640e27SKaricheri, Muralidharan module->name, ret); 28584640e27SKaricheri, Muralidharan list_del(&inst_modpriv->inst_list); 28684640e27SKaricheri, Muralidharan devm_kfree(dev, inst_modpriv); 28784640e27SKaricheri, Muralidharan return NETCP_MOD_PROBE_FAILED; 28884640e27SKaricheri, Muralidharan } 28984640e27SKaricheri, Muralidharan 29084640e27SKaricheri, Muralidharan /* Attach modules only if the primary module is probed */ 29184640e27SKaricheri, Muralidharan for_each_netcp_module(tmp) { 29284640e27SKaricheri, Muralidharan if (tmp->primary) 29384640e27SKaricheri, Muralidharan primary_module_registered = true; 29484640e27SKaricheri, Muralidharan } 29584640e27SKaricheri, Muralidharan 29684640e27SKaricheri, Muralidharan if (!primary_module_registered) 29784640e27SKaricheri, Muralidharan return 0; 29884640e27SKaricheri, Muralidharan 29984640e27SKaricheri, Muralidharan /* Attach module to interfaces */ 30084640e27SKaricheri, Muralidharan list_for_each_entry(netcp_intf, &netcp_device->interface_head, 30184640e27SKaricheri, Muralidharan interface_list) { 30284640e27SKaricheri, Muralidharan struct netcp_intf_modpriv *intf_modpriv; 30384640e27SKaricheri, Muralidharan 30484640e27SKaricheri, Muralidharan intf_modpriv = devm_kzalloc(dev, sizeof(*intf_modpriv), 30584640e27SKaricheri, Muralidharan GFP_KERNEL); 30684640e27SKaricheri, Muralidharan if (!intf_modpriv) 30784640e27SKaricheri, Muralidharan return -ENOMEM; 30884640e27SKaricheri, Muralidharan 30984640e27SKaricheri, Muralidharan interface = of_parse_phandle(netcp_intf->node_interface, 31084640e27SKaricheri, Muralidharan module->name, 0); 31184640e27SKaricheri, Muralidharan 312915c5857SKaricheri, Muralidharan if (!interface) { 313915c5857SKaricheri, Muralidharan devm_kfree(dev, intf_modpriv); 314915c5857SKaricheri, Muralidharan continue; 315915c5857SKaricheri, Muralidharan } 316915c5857SKaricheri, Muralidharan 31784640e27SKaricheri, Muralidharan intf_modpriv->netcp_priv = netcp_intf; 31884640e27SKaricheri, Muralidharan intf_modpriv->netcp_module = module; 31984640e27SKaricheri, Muralidharan list_add_tail(&intf_modpriv->intf_list, 32084640e27SKaricheri, Muralidharan &netcp_intf->module_head); 32184640e27SKaricheri, Muralidharan 32284640e27SKaricheri, Muralidharan ret = module->attach(inst_modpriv->module_priv, 32384640e27SKaricheri, Muralidharan netcp_intf->ndev, interface, 32484640e27SKaricheri, Muralidharan &intf_modpriv->module_priv); 32584640e27SKaricheri, Muralidharan of_node_put(interface); 32684640e27SKaricheri, Muralidharan if (ret) { 32784640e27SKaricheri, Muralidharan dev_dbg(dev, "Attach of module %s declined with %d\n", 32884640e27SKaricheri, Muralidharan module->name, ret); 32984640e27SKaricheri, Muralidharan list_del(&intf_modpriv->intf_list); 33084640e27SKaricheri, Muralidharan devm_kfree(dev, intf_modpriv); 33184640e27SKaricheri, Muralidharan continue; 33284640e27SKaricheri, Muralidharan } 33384640e27SKaricheri, Muralidharan } 334736532a0SKaricheri, Muralidharan 335736532a0SKaricheri, Muralidharan /* Now register the interface with netdev */ 336736532a0SKaricheri, Muralidharan list_for_each_entry(netcp_intf, 337736532a0SKaricheri, Muralidharan &netcp_device->interface_head, 338736532a0SKaricheri, Muralidharan interface_list) { 339736532a0SKaricheri, Muralidharan /* If interface not registered then register now */ 340736532a0SKaricheri, Muralidharan if (!netcp_intf->netdev_registered) { 341736532a0SKaricheri, Muralidharan ret = netcp_register_interface(netcp_intf); 342736532a0SKaricheri, Muralidharan if (ret) 343736532a0SKaricheri, Muralidharan return -ENODEV; 344736532a0SKaricheri, Muralidharan } 345736532a0SKaricheri, Muralidharan } 34684640e27SKaricheri, Muralidharan return 0; 34784640e27SKaricheri, Muralidharan } 34884640e27SKaricheri, Muralidharan 34984640e27SKaricheri, Muralidharan int netcp_register_module(struct netcp_module *module) 35084640e27SKaricheri, Muralidharan { 35184640e27SKaricheri, Muralidharan struct netcp_device *netcp_device; 35284640e27SKaricheri, Muralidharan struct netcp_module *tmp; 35384640e27SKaricheri, Muralidharan int ret; 35484640e27SKaricheri, Muralidharan 35584640e27SKaricheri, Muralidharan if (!module->name) { 35684640e27SKaricheri, Muralidharan WARN(1, "error registering netcp module: no name\n"); 35784640e27SKaricheri, Muralidharan return -EINVAL; 35884640e27SKaricheri, Muralidharan } 35984640e27SKaricheri, Muralidharan 36084640e27SKaricheri, Muralidharan if (!module->probe) { 36184640e27SKaricheri, Muralidharan WARN(1, "error registering netcp module: no probe\n"); 36284640e27SKaricheri, Muralidharan return -EINVAL; 36384640e27SKaricheri, Muralidharan } 36484640e27SKaricheri, Muralidharan 36584640e27SKaricheri, Muralidharan mutex_lock(&netcp_modules_lock); 36684640e27SKaricheri, Muralidharan 36784640e27SKaricheri, Muralidharan for_each_netcp_module(tmp) { 36884640e27SKaricheri, Muralidharan if (!strcasecmp(tmp->name, module->name)) { 36984640e27SKaricheri, Muralidharan mutex_unlock(&netcp_modules_lock); 37084640e27SKaricheri, Muralidharan return -EEXIST; 37184640e27SKaricheri, Muralidharan } 37284640e27SKaricheri, Muralidharan } 37384640e27SKaricheri, Muralidharan list_add_tail(&module->module_list, &netcp_modules); 37484640e27SKaricheri, Muralidharan 37584640e27SKaricheri, Muralidharan list_for_each_entry(netcp_device, &netcp_devices, device_list) { 37684640e27SKaricheri, Muralidharan ret = netcp_module_probe(netcp_device, module); 37784640e27SKaricheri, Muralidharan if (ret < 0) 37884640e27SKaricheri, Muralidharan goto fail; 37984640e27SKaricheri, Muralidharan } 38084640e27SKaricheri, Muralidharan mutex_unlock(&netcp_modules_lock); 38184640e27SKaricheri, Muralidharan return 0; 38284640e27SKaricheri, Muralidharan 38384640e27SKaricheri, Muralidharan fail: 38484640e27SKaricheri, Muralidharan mutex_unlock(&netcp_modules_lock); 38584640e27SKaricheri, Muralidharan netcp_unregister_module(module); 38684640e27SKaricheri, Muralidharan return ret; 38784640e27SKaricheri, Muralidharan } 38858c11b5fSKaricheri, Muralidharan EXPORT_SYMBOL_GPL(netcp_register_module); 38984640e27SKaricheri, Muralidharan 39084640e27SKaricheri, Muralidharan static void netcp_release_module(struct netcp_device *netcp_device, 39184640e27SKaricheri, Muralidharan struct netcp_module *module) 39284640e27SKaricheri, Muralidharan { 39384640e27SKaricheri, Muralidharan struct netcp_inst_modpriv *inst_modpriv, *inst_tmp; 39484640e27SKaricheri, Muralidharan struct netcp_intf *netcp_intf, *netcp_tmp; 39584640e27SKaricheri, Muralidharan struct device *dev = netcp_device->device; 39684640e27SKaricheri, Muralidharan 39784640e27SKaricheri, Muralidharan /* Release the module from each interface */ 39884640e27SKaricheri, Muralidharan list_for_each_entry_safe(netcp_intf, netcp_tmp, 39984640e27SKaricheri, Muralidharan &netcp_device->interface_head, 40084640e27SKaricheri, Muralidharan interface_list) { 40184640e27SKaricheri, Muralidharan struct netcp_intf_modpriv *intf_modpriv, *intf_tmp; 40284640e27SKaricheri, Muralidharan 40384640e27SKaricheri, Muralidharan list_for_each_entry_safe(intf_modpriv, intf_tmp, 40484640e27SKaricheri, Muralidharan &netcp_intf->module_head, 40584640e27SKaricheri, Muralidharan intf_list) { 40684640e27SKaricheri, Muralidharan if (intf_modpriv->netcp_module == module) { 40784640e27SKaricheri, Muralidharan module->release(intf_modpriv->module_priv); 40884640e27SKaricheri, Muralidharan list_del(&intf_modpriv->intf_list); 40984640e27SKaricheri, Muralidharan devm_kfree(dev, intf_modpriv); 41084640e27SKaricheri, Muralidharan break; 41184640e27SKaricheri, Muralidharan } 41284640e27SKaricheri, Muralidharan } 41384640e27SKaricheri, Muralidharan } 41484640e27SKaricheri, Muralidharan 41584640e27SKaricheri, Muralidharan /* Remove the module from each instance */ 41684640e27SKaricheri, Muralidharan list_for_each_entry_safe(inst_modpriv, inst_tmp, 41784640e27SKaricheri, Muralidharan &netcp_device->modpriv_head, inst_list) { 41884640e27SKaricheri, Muralidharan if (inst_modpriv->netcp_module == module) { 41984640e27SKaricheri, Muralidharan module->remove(netcp_device, 42084640e27SKaricheri, Muralidharan inst_modpriv->module_priv); 42184640e27SKaricheri, Muralidharan list_del(&inst_modpriv->inst_list); 42284640e27SKaricheri, Muralidharan devm_kfree(dev, inst_modpriv); 42384640e27SKaricheri, Muralidharan break; 42484640e27SKaricheri, Muralidharan } 42584640e27SKaricheri, Muralidharan } 42684640e27SKaricheri, Muralidharan } 42784640e27SKaricheri, Muralidharan 42884640e27SKaricheri, Muralidharan void netcp_unregister_module(struct netcp_module *module) 42984640e27SKaricheri, Muralidharan { 43084640e27SKaricheri, Muralidharan struct netcp_device *netcp_device; 43184640e27SKaricheri, Muralidharan struct netcp_module *module_tmp; 43284640e27SKaricheri, Muralidharan 43384640e27SKaricheri, Muralidharan mutex_lock(&netcp_modules_lock); 43484640e27SKaricheri, Muralidharan 43584640e27SKaricheri, Muralidharan list_for_each_entry(netcp_device, &netcp_devices, device_list) { 43684640e27SKaricheri, Muralidharan netcp_release_module(netcp_device, module); 43784640e27SKaricheri, Muralidharan } 43884640e27SKaricheri, Muralidharan 43984640e27SKaricheri, Muralidharan /* Remove the module from the module list */ 44084640e27SKaricheri, Muralidharan for_each_netcp_module(module_tmp) { 44184640e27SKaricheri, Muralidharan if (module == module_tmp) { 44284640e27SKaricheri, Muralidharan list_del(&module->module_list); 44384640e27SKaricheri, Muralidharan break; 44484640e27SKaricheri, Muralidharan } 44584640e27SKaricheri, Muralidharan } 44684640e27SKaricheri, Muralidharan 44784640e27SKaricheri, Muralidharan mutex_unlock(&netcp_modules_lock); 44884640e27SKaricheri, Muralidharan } 44958c11b5fSKaricheri, Muralidharan EXPORT_SYMBOL_GPL(netcp_unregister_module); 45084640e27SKaricheri, Muralidharan 45184640e27SKaricheri, Muralidharan void *netcp_module_get_intf_data(struct netcp_module *module, 45284640e27SKaricheri, Muralidharan struct netcp_intf *intf) 45384640e27SKaricheri, Muralidharan { 45484640e27SKaricheri, Muralidharan struct netcp_intf_modpriv *intf_modpriv; 45584640e27SKaricheri, Muralidharan 45684640e27SKaricheri, Muralidharan list_for_each_entry(intf_modpriv, &intf->module_head, intf_list) 45784640e27SKaricheri, Muralidharan if (intf_modpriv->netcp_module == module) 45884640e27SKaricheri, Muralidharan return intf_modpriv->module_priv; 45984640e27SKaricheri, Muralidharan return NULL; 46084640e27SKaricheri, Muralidharan } 46158c11b5fSKaricheri, Muralidharan EXPORT_SYMBOL_GPL(netcp_module_get_intf_data); 46284640e27SKaricheri, Muralidharan 46384640e27SKaricheri, Muralidharan /* Module TX and RX Hook management */ 46484640e27SKaricheri, Muralidharan struct netcp_hook_list { 46584640e27SKaricheri, Muralidharan struct list_head list; 46684640e27SKaricheri, Muralidharan netcp_hook_rtn *hook_rtn; 46784640e27SKaricheri, Muralidharan void *hook_data; 46884640e27SKaricheri, Muralidharan int order; 46984640e27SKaricheri, Muralidharan }; 47084640e27SKaricheri, Muralidharan 47184640e27SKaricheri, Muralidharan int netcp_register_txhook(struct netcp_intf *netcp_priv, int order, 47284640e27SKaricheri, Muralidharan netcp_hook_rtn *hook_rtn, void *hook_data) 47384640e27SKaricheri, Muralidharan { 47484640e27SKaricheri, Muralidharan struct netcp_hook_list *entry; 47584640e27SKaricheri, Muralidharan struct netcp_hook_list *next; 47684640e27SKaricheri, Muralidharan unsigned long flags; 47784640e27SKaricheri, Muralidharan 47884640e27SKaricheri, Muralidharan entry = devm_kzalloc(netcp_priv->dev, sizeof(*entry), GFP_KERNEL); 47984640e27SKaricheri, Muralidharan if (!entry) 48084640e27SKaricheri, Muralidharan return -ENOMEM; 48184640e27SKaricheri, Muralidharan 48284640e27SKaricheri, Muralidharan entry->hook_rtn = hook_rtn; 48384640e27SKaricheri, Muralidharan entry->hook_data = hook_data; 48484640e27SKaricheri, Muralidharan entry->order = order; 48584640e27SKaricheri, Muralidharan 48684640e27SKaricheri, Muralidharan spin_lock_irqsave(&netcp_priv->lock, flags); 48784640e27SKaricheri, Muralidharan list_for_each_entry(next, &netcp_priv->txhook_list_head, list) { 48884640e27SKaricheri, Muralidharan if (next->order > order) 48984640e27SKaricheri, Muralidharan break; 49084640e27SKaricheri, Muralidharan } 49184640e27SKaricheri, Muralidharan __list_add(&entry->list, next->list.prev, &next->list); 49284640e27SKaricheri, Muralidharan spin_unlock_irqrestore(&netcp_priv->lock, flags); 49384640e27SKaricheri, Muralidharan 49484640e27SKaricheri, Muralidharan return 0; 49584640e27SKaricheri, Muralidharan } 49658c11b5fSKaricheri, Muralidharan EXPORT_SYMBOL_GPL(netcp_register_txhook); 49784640e27SKaricheri, Muralidharan 49884640e27SKaricheri, Muralidharan int netcp_unregister_txhook(struct netcp_intf *netcp_priv, int order, 49984640e27SKaricheri, Muralidharan netcp_hook_rtn *hook_rtn, void *hook_data) 50084640e27SKaricheri, Muralidharan { 50184640e27SKaricheri, Muralidharan struct netcp_hook_list *next, *n; 50284640e27SKaricheri, Muralidharan unsigned long flags; 50384640e27SKaricheri, Muralidharan 50484640e27SKaricheri, Muralidharan spin_lock_irqsave(&netcp_priv->lock, flags); 50584640e27SKaricheri, Muralidharan list_for_each_entry_safe(next, n, &netcp_priv->txhook_list_head, list) { 50684640e27SKaricheri, Muralidharan if ((next->order == order) && 50784640e27SKaricheri, Muralidharan (next->hook_rtn == hook_rtn) && 50884640e27SKaricheri, Muralidharan (next->hook_data == hook_data)) { 50984640e27SKaricheri, Muralidharan list_del(&next->list); 51084640e27SKaricheri, Muralidharan spin_unlock_irqrestore(&netcp_priv->lock, flags); 51184640e27SKaricheri, Muralidharan devm_kfree(netcp_priv->dev, next); 51284640e27SKaricheri, Muralidharan return 0; 51384640e27SKaricheri, Muralidharan } 51484640e27SKaricheri, Muralidharan } 51584640e27SKaricheri, Muralidharan spin_unlock_irqrestore(&netcp_priv->lock, flags); 51684640e27SKaricheri, Muralidharan return -ENOENT; 51784640e27SKaricheri, Muralidharan } 51858c11b5fSKaricheri, Muralidharan EXPORT_SYMBOL_GPL(netcp_unregister_txhook); 51984640e27SKaricheri, Muralidharan 52084640e27SKaricheri, Muralidharan int netcp_register_rxhook(struct netcp_intf *netcp_priv, int order, 52184640e27SKaricheri, Muralidharan netcp_hook_rtn *hook_rtn, void *hook_data) 52284640e27SKaricheri, Muralidharan { 52384640e27SKaricheri, Muralidharan struct netcp_hook_list *entry; 52484640e27SKaricheri, Muralidharan struct netcp_hook_list *next; 52584640e27SKaricheri, Muralidharan unsigned long flags; 52684640e27SKaricheri, Muralidharan 52784640e27SKaricheri, Muralidharan entry = devm_kzalloc(netcp_priv->dev, sizeof(*entry), GFP_KERNEL); 52884640e27SKaricheri, Muralidharan if (!entry) 52984640e27SKaricheri, Muralidharan return -ENOMEM; 53084640e27SKaricheri, Muralidharan 53184640e27SKaricheri, Muralidharan entry->hook_rtn = hook_rtn; 53284640e27SKaricheri, Muralidharan entry->hook_data = hook_data; 53384640e27SKaricheri, Muralidharan entry->order = order; 53484640e27SKaricheri, Muralidharan 53584640e27SKaricheri, Muralidharan spin_lock_irqsave(&netcp_priv->lock, flags); 53684640e27SKaricheri, Muralidharan list_for_each_entry(next, &netcp_priv->rxhook_list_head, list) { 53784640e27SKaricheri, Muralidharan if (next->order > order) 53884640e27SKaricheri, Muralidharan break; 53984640e27SKaricheri, Muralidharan } 54084640e27SKaricheri, Muralidharan __list_add(&entry->list, next->list.prev, &next->list); 54184640e27SKaricheri, Muralidharan spin_unlock_irqrestore(&netcp_priv->lock, flags); 54284640e27SKaricheri, Muralidharan 54384640e27SKaricheri, Muralidharan return 0; 54484640e27SKaricheri, Muralidharan } 5456246168bSWingMan Kwok EXPORT_SYMBOL_GPL(netcp_register_rxhook); 54684640e27SKaricheri, Muralidharan 54784640e27SKaricheri, Muralidharan int netcp_unregister_rxhook(struct netcp_intf *netcp_priv, int order, 54884640e27SKaricheri, Muralidharan netcp_hook_rtn *hook_rtn, void *hook_data) 54984640e27SKaricheri, Muralidharan { 55084640e27SKaricheri, Muralidharan struct netcp_hook_list *next, *n; 55184640e27SKaricheri, Muralidharan unsigned long flags; 55284640e27SKaricheri, Muralidharan 55384640e27SKaricheri, Muralidharan spin_lock_irqsave(&netcp_priv->lock, flags); 55484640e27SKaricheri, Muralidharan list_for_each_entry_safe(next, n, &netcp_priv->rxhook_list_head, list) { 55584640e27SKaricheri, Muralidharan if ((next->order == order) && 55684640e27SKaricheri, Muralidharan (next->hook_rtn == hook_rtn) && 55784640e27SKaricheri, Muralidharan (next->hook_data == hook_data)) { 55884640e27SKaricheri, Muralidharan list_del(&next->list); 55984640e27SKaricheri, Muralidharan spin_unlock_irqrestore(&netcp_priv->lock, flags); 56084640e27SKaricheri, Muralidharan devm_kfree(netcp_priv->dev, next); 56184640e27SKaricheri, Muralidharan return 0; 56284640e27SKaricheri, Muralidharan } 56384640e27SKaricheri, Muralidharan } 56484640e27SKaricheri, Muralidharan spin_unlock_irqrestore(&netcp_priv->lock, flags); 56584640e27SKaricheri, Muralidharan 56684640e27SKaricheri, Muralidharan return -ENOENT; 56784640e27SKaricheri, Muralidharan } 5686246168bSWingMan Kwok EXPORT_SYMBOL_GPL(netcp_unregister_rxhook); 56984640e27SKaricheri, Muralidharan 57084640e27SKaricheri, Muralidharan static void netcp_frag_free(bool is_frag, void *ptr) 57184640e27SKaricheri, Muralidharan { 57284640e27SKaricheri, Muralidharan if (is_frag) 5737d525c4eSAlexander Duyck skb_free_frag(ptr); 57484640e27SKaricheri, Muralidharan else 57584640e27SKaricheri, Muralidharan kfree(ptr); 57684640e27SKaricheri, Muralidharan } 57784640e27SKaricheri, Muralidharan 57884640e27SKaricheri, Muralidharan static void netcp_free_rx_desc_chain(struct netcp_intf *netcp, 57984640e27SKaricheri, Muralidharan struct knav_dma_desc *desc) 58084640e27SKaricheri, Muralidharan { 58184640e27SKaricheri, Muralidharan struct knav_dma_desc *ndesc; 58284640e27SKaricheri, Muralidharan dma_addr_t dma_desc, dma_buf; 58384640e27SKaricheri, Muralidharan unsigned int buf_len, dma_sz = sizeof(*ndesc); 58484640e27SKaricheri, Muralidharan void *buf_ptr; 585958d104eSArnd Bergmann u32 tmp; 58684640e27SKaricheri, Muralidharan 58784640e27SKaricheri, Muralidharan get_words(&dma_desc, 1, &desc->next_desc); 58884640e27SKaricheri, Muralidharan 58984640e27SKaricheri, Muralidharan while (dma_desc) { 59084640e27SKaricheri, Muralidharan ndesc = knav_pool_desc_unmap(netcp->rx_pool, dma_desc, dma_sz); 59184640e27SKaricheri, Muralidharan if (unlikely(!ndesc)) { 59284640e27SKaricheri, Muralidharan dev_err(netcp->ndev_dev, "failed to unmap Rx desc\n"); 59384640e27SKaricheri, Muralidharan break; 59484640e27SKaricheri, Muralidharan } 595958d104eSArnd Bergmann get_pkt_info(&dma_buf, &tmp, &dma_desc, ndesc); 59606324481SKaricheri, Muralidharan /* warning!!!! We are retrieving the virtual ptr in the sw_data 59706324481SKaricheri, Muralidharan * field as a 32bit value. Will not work on 64bit machines 59806324481SKaricheri, Muralidharan */ 59906324481SKaricheri, Muralidharan buf_ptr = (void *)GET_SW_DATA0(ndesc); 60006324481SKaricheri, Muralidharan buf_len = (int)GET_SW_DATA1(desc); 60184640e27SKaricheri, Muralidharan dma_unmap_page(netcp->dev, dma_buf, PAGE_SIZE, DMA_FROM_DEVICE); 60284640e27SKaricheri, Muralidharan __free_page(buf_ptr); 60384640e27SKaricheri, Muralidharan knav_pool_desc_put(netcp->rx_pool, desc); 60484640e27SKaricheri, Muralidharan } 60506324481SKaricheri, Muralidharan /* warning!!!! We are retrieving the virtual ptr in the sw_data 60606324481SKaricheri, Muralidharan * field as a 32bit value. Will not work on 64bit machines 60706324481SKaricheri, Muralidharan */ 60806324481SKaricheri, Muralidharan buf_ptr = (void *)GET_SW_DATA0(desc); 60906324481SKaricheri, Muralidharan buf_len = (int)GET_SW_DATA1(desc); 61089907779SArnd Bergmann 61184640e27SKaricheri, Muralidharan if (buf_ptr) 61284640e27SKaricheri, Muralidharan netcp_frag_free(buf_len <= PAGE_SIZE, buf_ptr); 61384640e27SKaricheri, Muralidharan knav_pool_desc_put(netcp->rx_pool, desc); 61484640e27SKaricheri, Muralidharan } 61584640e27SKaricheri, Muralidharan 61684640e27SKaricheri, Muralidharan static void netcp_empty_rx_queue(struct netcp_intf *netcp) 61784640e27SKaricheri, Muralidharan { 6186a8162e9SMichael Scherban struct netcp_stats *rx_stats = &netcp->stats; 61984640e27SKaricheri, Muralidharan struct knav_dma_desc *desc; 62084640e27SKaricheri, Muralidharan unsigned int dma_sz; 62184640e27SKaricheri, Muralidharan dma_addr_t dma; 62284640e27SKaricheri, Muralidharan 62384640e27SKaricheri, Muralidharan for (; ;) { 62484640e27SKaricheri, Muralidharan dma = knav_queue_pop(netcp->rx_queue, &dma_sz); 62584640e27SKaricheri, Muralidharan if (!dma) 62684640e27SKaricheri, Muralidharan break; 62784640e27SKaricheri, Muralidharan 62884640e27SKaricheri, Muralidharan desc = knav_pool_desc_unmap(netcp->rx_pool, dma, dma_sz); 62984640e27SKaricheri, Muralidharan if (unlikely(!desc)) { 63084640e27SKaricheri, Muralidharan dev_err(netcp->ndev_dev, "%s: failed to unmap Rx desc\n", 63184640e27SKaricheri, Muralidharan __func__); 6326a8162e9SMichael Scherban rx_stats->rx_errors++; 63384640e27SKaricheri, Muralidharan continue; 63484640e27SKaricheri, Muralidharan } 63584640e27SKaricheri, Muralidharan netcp_free_rx_desc_chain(netcp, desc); 6366a8162e9SMichael Scherban rx_stats->rx_dropped++; 63784640e27SKaricheri, Muralidharan } 63884640e27SKaricheri, Muralidharan } 63984640e27SKaricheri, Muralidharan 64084640e27SKaricheri, Muralidharan static int netcp_process_one_rx_packet(struct netcp_intf *netcp) 64184640e27SKaricheri, Muralidharan { 6426a8162e9SMichael Scherban struct netcp_stats *rx_stats = &netcp->stats; 64384640e27SKaricheri, Muralidharan unsigned int dma_sz, buf_len, org_buf_len; 64484640e27SKaricheri, Muralidharan struct knav_dma_desc *desc, *ndesc; 64584640e27SKaricheri, Muralidharan unsigned int pkt_sz = 0, accum_sz; 64684640e27SKaricheri, Muralidharan struct netcp_hook_list *rx_hook; 64784640e27SKaricheri, Muralidharan dma_addr_t dma_desc, dma_buff; 64884640e27SKaricheri, Muralidharan struct netcp_packet p_info; 64984640e27SKaricheri, Muralidharan struct sk_buff *skb; 65084640e27SKaricheri, Muralidharan void *org_buf_ptr; 65169d707d0SKaricheri, Muralidharan u32 tmp; 65284640e27SKaricheri, Muralidharan 65384640e27SKaricheri, Muralidharan dma_desc = knav_queue_pop(netcp->rx_queue, &dma_sz); 65484640e27SKaricheri, Muralidharan if (!dma_desc) 65584640e27SKaricheri, Muralidharan return -1; 65684640e27SKaricheri, Muralidharan 65784640e27SKaricheri, Muralidharan desc = knav_pool_desc_unmap(netcp->rx_pool, dma_desc, dma_sz); 65884640e27SKaricheri, Muralidharan if (unlikely(!desc)) { 65984640e27SKaricheri, Muralidharan dev_err(netcp->ndev_dev, "failed to unmap Rx desc\n"); 66084640e27SKaricheri, Muralidharan return 0; 66184640e27SKaricheri, Muralidharan } 66284640e27SKaricheri, Muralidharan 66384640e27SKaricheri, Muralidharan get_pkt_info(&dma_buff, &buf_len, &dma_desc, desc); 66406324481SKaricheri, Muralidharan /* warning!!!! We are retrieving the virtual ptr in the sw_data 66506324481SKaricheri, Muralidharan * field as a 32bit value. Will not work on 64bit machines 66606324481SKaricheri, Muralidharan */ 66706324481SKaricheri, Muralidharan org_buf_ptr = (void *)GET_SW_DATA0(desc); 66806324481SKaricheri, Muralidharan org_buf_len = (int)GET_SW_DATA1(desc); 66984640e27SKaricheri, Muralidharan 67084640e27SKaricheri, Muralidharan if (unlikely(!org_buf_ptr)) { 67184640e27SKaricheri, Muralidharan dev_err(netcp->ndev_dev, "NULL bufptr in desc\n"); 67284640e27SKaricheri, Muralidharan goto free_desc; 67384640e27SKaricheri, Muralidharan } 67484640e27SKaricheri, Muralidharan 67584640e27SKaricheri, Muralidharan pkt_sz &= KNAV_DMA_DESC_PKT_LEN_MASK; 67684640e27SKaricheri, Muralidharan accum_sz = buf_len; 67784640e27SKaricheri, Muralidharan dma_unmap_single(netcp->dev, dma_buff, buf_len, DMA_FROM_DEVICE); 67884640e27SKaricheri, Muralidharan 67984640e27SKaricheri, Muralidharan /* Build a new sk_buff for the primary buffer */ 68084640e27SKaricheri, Muralidharan skb = build_skb(org_buf_ptr, org_buf_len); 68184640e27SKaricheri, Muralidharan if (unlikely(!skb)) { 68284640e27SKaricheri, Muralidharan dev_err(netcp->ndev_dev, "build_skb() failed\n"); 68384640e27SKaricheri, Muralidharan goto free_desc; 68484640e27SKaricheri, Muralidharan } 68584640e27SKaricheri, Muralidharan 68684640e27SKaricheri, Muralidharan /* update data, tail and len */ 68784640e27SKaricheri, Muralidharan skb_reserve(skb, NETCP_SOP_OFFSET); 68884640e27SKaricheri, Muralidharan __skb_put(skb, buf_len); 68984640e27SKaricheri, Muralidharan 69084640e27SKaricheri, Muralidharan /* Fill in the page fragment list */ 69184640e27SKaricheri, Muralidharan while (dma_desc) { 69284640e27SKaricheri, Muralidharan struct page *page; 69384640e27SKaricheri, Muralidharan 69484640e27SKaricheri, Muralidharan ndesc = knav_pool_desc_unmap(netcp->rx_pool, dma_desc, dma_sz); 69584640e27SKaricheri, Muralidharan if (unlikely(!ndesc)) { 69684640e27SKaricheri, Muralidharan dev_err(netcp->ndev_dev, "failed to unmap Rx desc\n"); 69784640e27SKaricheri, Muralidharan goto free_desc; 69884640e27SKaricheri, Muralidharan } 69984640e27SKaricheri, Muralidharan 70084640e27SKaricheri, Muralidharan get_pkt_info(&dma_buff, &buf_len, &dma_desc, ndesc); 70106324481SKaricheri, Muralidharan /* warning!!!! We are retrieving the virtual ptr in the sw_data 70206324481SKaricheri, Muralidharan * field as a 32bit value. Will not work on 64bit machines 70306324481SKaricheri, Muralidharan */ 7045a717843SRex Chang page = (struct page *)GET_SW_DATA0(ndesc); 70584640e27SKaricheri, Muralidharan 70684640e27SKaricheri, Muralidharan if (likely(dma_buff && buf_len && page)) { 70784640e27SKaricheri, Muralidharan dma_unmap_page(netcp->dev, dma_buff, PAGE_SIZE, 70884640e27SKaricheri, Muralidharan DMA_FROM_DEVICE); 70984640e27SKaricheri, Muralidharan } else { 71089907779SArnd Bergmann dev_err(netcp->ndev_dev, "Bad Rx desc dma_buff(%pad), len(%d), page(%p)\n", 71189907779SArnd Bergmann &dma_buff, buf_len, page); 71284640e27SKaricheri, Muralidharan goto free_desc; 71384640e27SKaricheri, Muralidharan } 71484640e27SKaricheri, Muralidharan 71584640e27SKaricheri, Muralidharan skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page, 71684640e27SKaricheri, Muralidharan offset_in_page(dma_buff), buf_len, PAGE_SIZE); 71784640e27SKaricheri, Muralidharan accum_sz += buf_len; 71884640e27SKaricheri, Muralidharan 71984640e27SKaricheri, Muralidharan /* Free the descriptor */ 72084640e27SKaricheri, Muralidharan knav_pool_desc_put(netcp->rx_pool, ndesc); 72184640e27SKaricheri, Muralidharan } 72284640e27SKaricheri, Muralidharan 72384640e27SKaricheri, Muralidharan /* check for packet len and warn */ 72484640e27SKaricheri, Muralidharan if (unlikely(pkt_sz != accum_sz)) 72584640e27SKaricheri, Muralidharan dev_dbg(netcp->ndev_dev, "mismatch in packet size(%d) & sum of fragments(%d)\n", 72684640e27SKaricheri, Muralidharan pkt_sz, accum_sz); 72784640e27SKaricheri, Muralidharan 7284cd85a61SKaricheri, Muralidharan /* Newer version of the Ethernet switch can trim the Ethernet FCS 7294cd85a61SKaricheri, Muralidharan * from the packet and is indicated in hw_cap. So trim it only for 7304cd85a61SKaricheri, Muralidharan * older h/w 7314cd85a61SKaricheri, Muralidharan */ 7324cd85a61SKaricheri, Muralidharan if (!(netcp->hw_cap & ETH_SW_CAN_REMOVE_ETH_FCS)) 73384640e27SKaricheri, Muralidharan __pskb_trim(skb, skb->len - ETH_FCS_LEN); 73484640e27SKaricheri, Muralidharan 73584640e27SKaricheri, Muralidharan /* Call each of the RX hooks */ 73684640e27SKaricheri, Muralidharan p_info.skb = skb; 7376246168bSWingMan Kwok skb->dev = netcp->ndev; 73884640e27SKaricheri, Muralidharan p_info.rxtstamp_complete = false; 73969d707d0SKaricheri, Muralidharan get_desc_info(&tmp, &p_info.eflags, desc); 74069d707d0SKaricheri, Muralidharan p_info.epib = desc->epib; 74169d707d0SKaricheri, Muralidharan p_info.psdata = (u32 __force *)desc->psdata; 74269d707d0SKaricheri, Muralidharan p_info.eflags = ((p_info.eflags >> KNAV_DMA_DESC_EFLAGS_SHIFT) & 74369d707d0SKaricheri, Muralidharan KNAV_DMA_DESC_EFLAGS_MASK); 74484640e27SKaricheri, Muralidharan list_for_each_entry(rx_hook, &netcp->rxhook_list_head, list) { 74584640e27SKaricheri, Muralidharan int ret; 74684640e27SKaricheri, Muralidharan 74784640e27SKaricheri, Muralidharan ret = rx_hook->hook_rtn(rx_hook->order, rx_hook->hook_data, 74884640e27SKaricheri, Muralidharan &p_info); 74984640e27SKaricheri, Muralidharan if (unlikely(ret)) { 75084640e27SKaricheri, Muralidharan dev_err(netcp->ndev_dev, "RX hook %d failed: %d\n", 75184640e27SKaricheri, Muralidharan rx_hook->order, ret); 75269d707d0SKaricheri, Muralidharan /* Free the primary descriptor */ 7536a8162e9SMichael Scherban rx_stats->rx_dropped++; 75469d707d0SKaricheri, Muralidharan knav_pool_desc_put(netcp->rx_pool, desc); 75584640e27SKaricheri, Muralidharan dev_kfree_skb(skb); 75684640e27SKaricheri, Muralidharan return 0; 75784640e27SKaricheri, Muralidharan } 75884640e27SKaricheri, Muralidharan } 75969d707d0SKaricheri, Muralidharan /* Free the primary descriptor */ 76069d707d0SKaricheri, Muralidharan knav_pool_desc_put(netcp->rx_pool, desc); 76184640e27SKaricheri, Muralidharan 7626a8162e9SMichael Scherban u64_stats_update_begin(&rx_stats->syncp_rx); 7636a8162e9SMichael Scherban rx_stats->rx_packets++; 7646a8162e9SMichael Scherban rx_stats->rx_bytes += skb->len; 7656a8162e9SMichael Scherban u64_stats_update_end(&rx_stats->syncp_rx); 76684640e27SKaricheri, Muralidharan 76784640e27SKaricheri, Muralidharan /* push skb up the stack */ 76884640e27SKaricheri, Muralidharan skb->protocol = eth_type_trans(skb, netcp->ndev); 76984640e27SKaricheri, Muralidharan netif_receive_skb(skb); 77084640e27SKaricheri, Muralidharan return 0; 77184640e27SKaricheri, Muralidharan 77284640e27SKaricheri, Muralidharan free_desc: 77384640e27SKaricheri, Muralidharan netcp_free_rx_desc_chain(netcp, desc); 7746a8162e9SMichael Scherban rx_stats->rx_errors++; 77584640e27SKaricheri, Muralidharan return 0; 77684640e27SKaricheri, Muralidharan } 77784640e27SKaricheri, Muralidharan 77884640e27SKaricheri, Muralidharan static int netcp_process_rx_packets(struct netcp_intf *netcp, 77984640e27SKaricheri, Muralidharan unsigned int budget) 78084640e27SKaricheri, Muralidharan { 78184640e27SKaricheri, Muralidharan int i; 78284640e27SKaricheri, Muralidharan 78384640e27SKaricheri, Muralidharan for (i = 0; (i < budget) && !netcp_process_one_rx_packet(netcp); i++) 78484640e27SKaricheri, Muralidharan ; 78584640e27SKaricheri, Muralidharan return i; 78684640e27SKaricheri, Muralidharan } 78784640e27SKaricheri, Muralidharan 78884640e27SKaricheri, Muralidharan /* Release descriptors and attached buffers from Rx FDQ */ 78984640e27SKaricheri, Muralidharan static void netcp_free_rx_buf(struct netcp_intf *netcp, int fdq) 79084640e27SKaricheri, Muralidharan { 79184640e27SKaricheri, Muralidharan struct knav_dma_desc *desc; 79284640e27SKaricheri, Muralidharan unsigned int buf_len, dma_sz; 79384640e27SKaricheri, Muralidharan dma_addr_t dma; 79484640e27SKaricheri, Muralidharan void *buf_ptr; 79584640e27SKaricheri, Muralidharan 79684640e27SKaricheri, Muralidharan /* Allocate descriptor */ 79784640e27SKaricheri, Muralidharan while ((dma = knav_queue_pop(netcp->rx_fdq[fdq], &dma_sz))) { 79884640e27SKaricheri, Muralidharan desc = knav_pool_desc_unmap(netcp->rx_pool, dma, dma_sz); 79984640e27SKaricheri, Muralidharan if (unlikely(!desc)) { 80084640e27SKaricheri, Muralidharan dev_err(netcp->ndev_dev, "failed to unmap Rx desc\n"); 80184640e27SKaricheri, Muralidharan continue; 80284640e27SKaricheri, Muralidharan } 80384640e27SKaricheri, Muralidharan 80484640e27SKaricheri, Muralidharan get_org_pkt_info(&dma, &buf_len, desc); 80506324481SKaricheri, Muralidharan /* warning!!!! We are retrieving the virtual ptr in the sw_data 80606324481SKaricheri, Muralidharan * field as a 32bit value. Will not work on 64bit machines 80706324481SKaricheri, Muralidharan */ 80806324481SKaricheri, Muralidharan buf_ptr = (void *)GET_SW_DATA0(desc); 80984640e27SKaricheri, Muralidharan 81084640e27SKaricheri, Muralidharan if (unlikely(!dma)) { 81184640e27SKaricheri, Muralidharan dev_err(netcp->ndev_dev, "NULL orig_buff in desc\n"); 81284640e27SKaricheri, Muralidharan knav_pool_desc_put(netcp->rx_pool, desc); 81384640e27SKaricheri, Muralidharan continue; 81484640e27SKaricheri, Muralidharan } 81584640e27SKaricheri, Muralidharan 81684640e27SKaricheri, Muralidharan if (unlikely(!buf_ptr)) { 81784640e27SKaricheri, Muralidharan dev_err(netcp->ndev_dev, "NULL bufptr in desc\n"); 81884640e27SKaricheri, Muralidharan knav_pool_desc_put(netcp->rx_pool, desc); 81984640e27SKaricheri, Muralidharan continue; 82084640e27SKaricheri, Muralidharan } 82184640e27SKaricheri, Muralidharan 82284640e27SKaricheri, Muralidharan if (fdq == 0) { 82384640e27SKaricheri, Muralidharan dma_unmap_single(netcp->dev, dma, buf_len, 82484640e27SKaricheri, Muralidharan DMA_FROM_DEVICE); 82584640e27SKaricheri, Muralidharan netcp_frag_free((buf_len <= PAGE_SIZE), buf_ptr); 82684640e27SKaricheri, Muralidharan } else { 82784640e27SKaricheri, Muralidharan dma_unmap_page(netcp->dev, dma, buf_len, 82884640e27SKaricheri, Muralidharan DMA_FROM_DEVICE); 82984640e27SKaricheri, Muralidharan __free_page(buf_ptr); 83084640e27SKaricheri, Muralidharan } 83184640e27SKaricheri, Muralidharan 83284640e27SKaricheri, Muralidharan knav_pool_desc_put(netcp->rx_pool, desc); 83384640e27SKaricheri, Muralidharan } 83484640e27SKaricheri, Muralidharan } 83584640e27SKaricheri, Muralidharan 83684640e27SKaricheri, Muralidharan static void netcp_rxpool_free(struct netcp_intf *netcp) 83784640e27SKaricheri, Muralidharan { 83884640e27SKaricheri, Muralidharan int i; 83984640e27SKaricheri, Muralidharan 84084640e27SKaricheri, Muralidharan for (i = 0; i < KNAV_DMA_FDQ_PER_CHAN && 84184640e27SKaricheri, Muralidharan !IS_ERR_OR_NULL(netcp->rx_fdq[i]); i++) 84284640e27SKaricheri, Muralidharan netcp_free_rx_buf(netcp, i); 84384640e27SKaricheri, Muralidharan 84484640e27SKaricheri, Muralidharan if (knav_pool_count(netcp->rx_pool) != netcp->rx_pool_size) 84584640e27SKaricheri, Muralidharan dev_err(netcp->ndev_dev, "Lost Rx (%d) descriptors\n", 84684640e27SKaricheri, Muralidharan netcp->rx_pool_size - knav_pool_count(netcp->rx_pool)); 84784640e27SKaricheri, Muralidharan 84884640e27SKaricheri, Muralidharan knav_pool_destroy(netcp->rx_pool); 84984640e27SKaricheri, Muralidharan netcp->rx_pool = NULL; 85084640e27SKaricheri, Muralidharan } 85184640e27SKaricheri, Muralidharan 852e558b1fbSKaricheri, Muralidharan static int netcp_allocate_rx_buf(struct netcp_intf *netcp, int fdq) 85384640e27SKaricheri, Muralidharan { 85484640e27SKaricheri, Muralidharan struct knav_dma_desc *hwdesc; 85584640e27SKaricheri, Muralidharan unsigned int buf_len, dma_sz; 85684640e27SKaricheri, Muralidharan u32 desc_info, pkt_info; 85784640e27SKaricheri, Muralidharan struct page *page; 85884640e27SKaricheri, Muralidharan dma_addr_t dma; 85984640e27SKaricheri, Muralidharan void *bufptr; 860b1cb86aeSKaricheri, Muralidharan u32 sw_data[2]; 86184640e27SKaricheri, Muralidharan 86284640e27SKaricheri, Muralidharan /* Allocate descriptor */ 86384640e27SKaricheri, Muralidharan hwdesc = knav_pool_desc_get(netcp->rx_pool); 86484640e27SKaricheri, Muralidharan if (IS_ERR_OR_NULL(hwdesc)) { 86584640e27SKaricheri, Muralidharan dev_dbg(netcp->ndev_dev, "out of rx pool desc\n"); 866e558b1fbSKaricheri, Muralidharan return -ENOMEM; 86784640e27SKaricheri, Muralidharan } 86884640e27SKaricheri, Muralidharan 86984640e27SKaricheri, Muralidharan if (likely(fdq == 0)) { 87084640e27SKaricheri, Muralidharan unsigned int primary_buf_len; 87184640e27SKaricheri, Muralidharan /* Allocate a primary receive queue entry */ 872866b8b18SWingMan Kwok buf_len = NETCP_PACKET_SIZE + NETCP_SOP_OFFSET; 87384640e27SKaricheri, Muralidharan primary_buf_len = SKB_DATA_ALIGN(buf_len) + 87484640e27SKaricheri, Muralidharan SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 87584640e27SKaricheri, Muralidharan 87684640e27SKaricheri, Muralidharan bufptr = netdev_alloc_frag(primary_buf_len); 877b1cb86aeSKaricheri, Muralidharan sw_data[1] = primary_buf_len; 87884640e27SKaricheri, Muralidharan 87984640e27SKaricheri, Muralidharan if (unlikely(!bufptr)) { 880866b8b18SWingMan Kwok dev_warn_ratelimited(netcp->ndev_dev, 881866b8b18SWingMan Kwok "Primary RX buffer alloc failed\n"); 88284640e27SKaricheri, Muralidharan goto fail; 88384640e27SKaricheri, Muralidharan } 88484640e27SKaricheri, Muralidharan dma = dma_map_single(netcp->dev, bufptr, buf_len, 88584640e27SKaricheri, Muralidharan DMA_TO_DEVICE); 886866b8b18SWingMan Kwok if (unlikely(dma_mapping_error(netcp->dev, dma))) 887866b8b18SWingMan Kwok goto fail; 888866b8b18SWingMan Kwok 88906324481SKaricheri, Muralidharan /* warning!!!! We are saving the virtual ptr in the sw_data 89006324481SKaricheri, Muralidharan * field as a 32bit value. Will not work on 64bit machines 89106324481SKaricheri, Muralidharan */ 892b1cb86aeSKaricheri, Muralidharan sw_data[0] = (u32)bufptr; 89384640e27SKaricheri, Muralidharan } else { 89484640e27SKaricheri, Muralidharan /* Allocate a secondary receive queue entry */ 895453f85d4SMel Gorman page = alloc_page(GFP_ATOMIC | GFP_DMA); 89684640e27SKaricheri, Muralidharan if (unlikely(!page)) { 89784640e27SKaricheri, Muralidharan dev_warn_ratelimited(netcp->ndev_dev, "Secondary page alloc failed\n"); 89884640e27SKaricheri, Muralidharan goto fail; 89984640e27SKaricheri, Muralidharan } 90084640e27SKaricheri, Muralidharan buf_len = PAGE_SIZE; 90184640e27SKaricheri, Muralidharan dma = dma_map_page(netcp->dev, page, 0, buf_len, DMA_TO_DEVICE); 90206324481SKaricheri, Muralidharan /* warning!!!! We are saving the virtual ptr in the sw_data 90306324481SKaricheri, Muralidharan * field as a 32bit value. Will not work on 64bit machines 90406324481SKaricheri, Muralidharan */ 905b1cb86aeSKaricheri, Muralidharan sw_data[0] = (u32)page; 906b1cb86aeSKaricheri, Muralidharan sw_data[1] = 0; 90784640e27SKaricheri, Muralidharan } 90884640e27SKaricheri, Muralidharan 90984640e27SKaricheri, Muralidharan desc_info = KNAV_DMA_DESC_PS_INFO_IN_DESC; 91084640e27SKaricheri, Muralidharan desc_info |= buf_len & KNAV_DMA_DESC_PKT_LEN_MASK; 91184640e27SKaricheri, Muralidharan pkt_info = KNAV_DMA_DESC_HAS_EPIB; 91284640e27SKaricheri, Muralidharan pkt_info |= KNAV_DMA_NUM_PS_WORDS << KNAV_DMA_DESC_PSLEN_SHIFT; 91384640e27SKaricheri, Muralidharan pkt_info |= (netcp->rx_queue_id & KNAV_DMA_DESC_RETQ_MASK) << 91484640e27SKaricheri, Muralidharan KNAV_DMA_DESC_RETQ_SHIFT; 91584640e27SKaricheri, Muralidharan set_org_pkt_info(dma, buf_len, hwdesc); 91606324481SKaricheri, Muralidharan SET_SW_DATA0(sw_data[0], hwdesc); 91706324481SKaricheri, Muralidharan SET_SW_DATA1(sw_data[1], hwdesc); 91884640e27SKaricheri, Muralidharan set_desc_info(desc_info, pkt_info, hwdesc); 91984640e27SKaricheri, Muralidharan 92084640e27SKaricheri, Muralidharan /* Push to FDQs */ 92184640e27SKaricheri, Muralidharan knav_pool_desc_map(netcp->rx_pool, hwdesc, sizeof(*hwdesc), &dma, 92284640e27SKaricheri, Muralidharan &dma_sz); 92384640e27SKaricheri, Muralidharan knav_queue_push(netcp->rx_fdq[fdq], dma, sizeof(*hwdesc), 0); 924e558b1fbSKaricheri, Muralidharan return 0; 92584640e27SKaricheri, Muralidharan 92684640e27SKaricheri, Muralidharan fail: 92784640e27SKaricheri, Muralidharan knav_pool_desc_put(netcp->rx_pool, hwdesc); 928e558b1fbSKaricheri, Muralidharan return -ENOMEM; 92984640e27SKaricheri, Muralidharan } 93084640e27SKaricheri, Muralidharan 93184640e27SKaricheri, Muralidharan /* Refill Rx FDQ with descriptors & attached buffers */ 93284640e27SKaricheri, Muralidharan static void netcp_rxpool_refill(struct netcp_intf *netcp) 93384640e27SKaricheri, Muralidharan { 93484640e27SKaricheri, Muralidharan u32 fdq_deficit[KNAV_DMA_FDQ_PER_CHAN] = {0}; 935e558b1fbSKaricheri, Muralidharan int i, ret = 0; 93684640e27SKaricheri, Muralidharan 93784640e27SKaricheri, Muralidharan /* Calculate the FDQ deficit and refill */ 93884640e27SKaricheri, Muralidharan for (i = 0; i < KNAV_DMA_FDQ_PER_CHAN && netcp->rx_fdq[i]; i++) { 93984640e27SKaricheri, Muralidharan fdq_deficit[i] = netcp->rx_queue_depths[i] - 94084640e27SKaricheri, Muralidharan knav_queue_get_count(netcp->rx_fdq[i]); 94184640e27SKaricheri, Muralidharan 942e558b1fbSKaricheri, Muralidharan while (fdq_deficit[i]-- && !ret) 943e558b1fbSKaricheri, Muralidharan ret = netcp_allocate_rx_buf(netcp, i); 94484640e27SKaricheri, Muralidharan } /* end for fdqs */ 94584640e27SKaricheri, Muralidharan } 94684640e27SKaricheri, Muralidharan 94784640e27SKaricheri, Muralidharan /* NAPI poll */ 94884640e27SKaricheri, Muralidharan static int netcp_rx_poll(struct napi_struct *napi, int budget) 94984640e27SKaricheri, Muralidharan { 95084640e27SKaricheri, Muralidharan struct netcp_intf *netcp = container_of(napi, struct netcp_intf, 95184640e27SKaricheri, Muralidharan rx_napi); 95284640e27SKaricheri, Muralidharan unsigned int packets; 95384640e27SKaricheri, Muralidharan 95484640e27SKaricheri, Muralidharan packets = netcp_process_rx_packets(netcp, budget); 95584640e27SKaricheri, Muralidharan 95699f8ef5dSKaricheri, Muralidharan netcp_rxpool_refill(netcp); 95784640e27SKaricheri, Muralidharan if (packets < budget) { 9586ad20165SEric Dumazet napi_complete_done(&netcp->rx_napi, packets); 95984640e27SKaricheri, Muralidharan knav_queue_enable_notify(netcp->rx_queue); 96084640e27SKaricheri, Muralidharan } 96184640e27SKaricheri, Muralidharan 96284640e27SKaricheri, Muralidharan return packets; 96384640e27SKaricheri, Muralidharan } 96484640e27SKaricheri, Muralidharan 96584640e27SKaricheri, Muralidharan static void netcp_rx_notify(void *arg) 96684640e27SKaricheri, Muralidharan { 96784640e27SKaricheri, Muralidharan struct netcp_intf *netcp = arg; 96884640e27SKaricheri, Muralidharan 96984640e27SKaricheri, Muralidharan knav_queue_disable_notify(netcp->rx_queue); 97084640e27SKaricheri, Muralidharan napi_schedule(&netcp->rx_napi); 97184640e27SKaricheri, Muralidharan } 97284640e27SKaricheri, Muralidharan 97384640e27SKaricheri, Muralidharan static void netcp_free_tx_desc_chain(struct netcp_intf *netcp, 97484640e27SKaricheri, Muralidharan struct knav_dma_desc *desc, 97584640e27SKaricheri, Muralidharan unsigned int desc_sz) 97684640e27SKaricheri, Muralidharan { 97784640e27SKaricheri, Muralidharan struct knav_dma_desc *ndesc = desc; 97884640e27SKaricheri, Muralidharan dma_addr_t dma_desc, dma_buf; 97984640e27SKaricheri, Muralidharan unsigned int buf_len; 98084640e27SKaricheri, Muralidharan 98184640e27SKaricheri, Muralidharan while (ndesc) { 98284640e27SKaricheri, Muralidharan get_pkt_info(&dma_buf, &buf_len, &dma_desc, ndesc); 98384640e27SKaricheri, Muralidharan 98484640e27SKaricheri, Muralidharan if (dma_buf && buf_len) 98584640e27SKaricheri, Muralidharan dma_unmap_single(netcp->dev, dma_buf, buf_len, 98684640e27SKaricheri, Muralidharan DMA_TO_DEVICE); 98784640e27SKaricheri, Muralidharan else 98889907779SArnd Bergmann dev_warn(netcp->ndev_dev, "bad Tx desc buf(%pad), len(%d)\n", 98989907779SArnd Bergmann &dma_buf, buf_len); 99084640e27SKaricheri, Muralidharan 99184640e27SKaricheri, Muralidharan knav_pool_desc_put(netcp->tx_pool, ndesc); 99284640e27SKaricheri, Muralidharan ndesc = NULL; 99384640e27SKaricheri, Muralidharan if (dma_desc) { 99484640e27SKaricheri, Muralidharan ndesc = knav_pool_desc_unmap(netcp->tx_pool, dma_desc, 99584640e27SKaricheri, Muralidharan desc_sz); 99684640e27SKaricheri, Muralidharan if (!ndesc) 99784640e27SKaricheri, Muralidharan dev_err(netcp->ndev_dev, "failed to unmap Tx desc\n"); 99884640e27SKaricheri, Muralidharan } 99984640e27SKaricheri, Muralidharan } 100084640e27SKaricheri, Muralidharan } 100184640e27SKaricheri, Muralidharan 100284640e27SKaricheri, Muralidharan static int netcp_process_tx_compl_packets(struct netcp_intf *netcp, 100384640e27SKaricheri, Muralidharan unsigned int budget) 100484640e27SKaricheri, Muralidharan { 10056a8162e9SMichael Scherban struct netcp_stats *tx_stats = &netcp->stats; 100684640e27SKaricheri, Muralidharan struct knav_dma_desc *desc; 10076246168bSWingMan Kwok struct netcp_tx_cb *tx_cb; 100884640e27SKaricheri, Muralidharan struct sk_buff *skb; 100984640e27SKaricheri, Muralidharan unsigned int dma_sz; 101084640e27SKaricheri, Muralidharan dma_addr_t dma; 101184640e27SKaricheri, Muralidharan int pkts = 0; 101284640e27SKaricheri, Muralidharan 101384640e27SKaricheri, Muralidharan while (budget--) { 101484640e27SKaricheri, Muralidharan dma = knav_queue_pop(netcp->tx_compl_q, &dma_sz); 101584640e27SKaricheri, Muralidharan if (!dma) 101684640e27SKaricheri, Muralidharan break; 101784640e27SKaricheri, Muralidharan desc = knav_pool_desc_unmap(netcp->tx_pool, dma, dma_sz); 101884640e27SKaricheri, Muralidharan if (unlikely(!desc)) { 101984640e27SKaricheri, Muralidharan dev_err(netcp->ndev_dev, "failed to unmap Tx desc\n"); 10206a8162e9SMichael Scherban tx_stats->tx_errors++; 102184640e27SKaricheri, Muralidharan continue; 102284640e27SKaricheri, Muralidharan } 102384640e27SKaricheri, Muralidharan 102406324481SKaricheri, Muralidharan /* warning!!!! We are retrieving the virtual ptr in the sw_data 102506324481SKaricheri, Muralidharan * field as a 32bit value. Will not work on 64bit machines 102606324481SKaricheri, Muralidharan */ 102706324481SKaricheri, Muralidharan skb = (struct sk_buff *)GET_SW_DATA0(desc); 102884640e27SKaricheri, Muralidharan netcp_free_tx_desc_chain(netcp, desc, dma_sz); 102984640e27SKaricheri, Muralidharan if (!skb) { 103084640e27SKaricheri, Muralidharan dev_err(netcp->ndev_dev, "No skb in Tx desc\n"); 10316a8162e9SMichael Scherban tx_stats->tx_errors++; 103284640e27SKaricheri, Muralidharan continue; 103384640e27SKaricheri, Muralidharan } 103484640e27SKaricheri, Muralidharan 10356246168bSWingMan Kwok tx_cb = (struct netcp_tx_cb *)skb->cb; 10366246168bSWingMan Kwok if (tx_cb->txtstamp) 10376246168bSWingMan Kwok tx_cb->txtstamp(tx_cb->ts_context, skb); 10386246168bSWingMan Kwok 103984640e27SKaricheri, Muralidharan if (netif_subqueue_stopped(netcp->ndev, skb) && 104084640e27SKaricheri, Muralidharan netif_running(netcp->ndev) && 104184640e27SKaricheri, Muralidharan (knav_pool_count(netcp->tx_pool) > 104284640e27SKaricheri, Muralidharan netcp->tx_resume_threshold)) { 104384640e27SKaricheri, Muralidharan u16 subqueue = skb_get_queue_mapping(skb); 104484640e27SKaricheri, Muralidharan 104584640e27SKaricheri, Muralidharan netif_wake_subqueue(netcp->ndev, subqueue); 104684640e27SKaricheri, Muralidharan } 104784640e27SKaricheri, Muralidharan 10486a8162e9SMichael Scherban u64_stats_update_begin(&tx_stats->syncp_tx); 10496a8162e9SMichael Scherban tx_stats->tx_packets++; 10506a8162e9SMichael Scherban tx_stats->tx_bytes += skb->len; 10516a8162e9SMichael Scherban u64_stats_update_end(&tx_stats->syncp_tx); 105284640e27SKaricheri, Muralidharan dev_kfree_skb(skb); 105384640e27SKaricheri, Muralidharan pkts++; 105484640e27SKaricheri, Muralidharan } 105584640e27SKaricheri, Muralidharan return pkts; 105684640e27SKaricheri, Muralidharan } 105784640e27SKaricheri, Muralidharan 105884640e27SKaricheri, Muralidharan static int netcp_tx_poll(struct napi_struct *napi, int budget) 105984640e27SKaricheri, Muralidharan { 106084640e27SKaricheri, Muralidharan int packets; 106184640e27SKaricheri, Muralidharan struct netcp_intf *netcp = container_of(napi, struct netcp_intf, 106284640e27SKaricheri, Muralidharan tx_napi); 106384640e27SKaricheri, Muralidharan 106484640e27SKaricheri, Muralidharan packets = netcp_process_tx_compl_packets(netcp, budget); 106584640e27SKaricheri, Muralidharan if (packets < budget) { 106684640e27SKaricheri, Muralidharan napi_complete(&netcp->tx_napi); 106784640e27SKaricheri, Muralidharan knav_queue_enable_notify(netcp->tx_compl_q); 106884640e27SKaricheri, Muralidharan } 106984640e27SKaricheri, Muralidharan 107084640e27SKaricheri, Muralidharan return packets; 107184640e27SKaricheri, Muralidharan } 107284640e27SKaricheri, Muralidharan 107384640e27SKaricheri, Muralidharan static void netcp_tx_notify(void *arg) 107484640e27SKaricheri, Muralidharan { 107584640e27SKaricheri, Muralidharan struct netcp_intf *netcp = arg; 107684640e27SKaricheri, Muralidharan 107784640e27SKaricheri, Muralidharan knav_queue_disable_notify(netcp->tx_compl_q); 107884640e27SKaricheri, Muralidharan napi_schedule(&netcp->tx_napi); 107984640e27SKaricheri, Muralidharan } 108084640e27SKaricheri, Muralidharan 108184640e27SKaricheri, Muralidharan static struct knav_dma_desc* 108284640e27SKaricheri, Muralidharan netcp_tx_map_skb(struct sk_buff *skb, struct netcp_intf *netcp) 108384640e27SKaricheri, Muralidharan { 108484640e27SKaricheri, Muralidharan struct knav_dma_desc *desc, *ndesc, *pdesc; 108584640e27SKaricheri, Muralidharan unsigned int pkt_len = skb_headlen(skb); 108684640e27SKaricheri, Muralidharan struct device *dev = netcp->dev; 108784640e27SKaricheri, Muralidharan dma_addr_t dma_addr; 108884640e27SKaricheri, Muralidharan unsigned int dma_sz; 108984640e27SKaricheri, Muralidharan int i; 109084640e27SKaricheri, Muralidharan 109184640e27SKaricheri, Muralidharan /* Map the linear buffer */ 109284640e27SKaricheri, Muralidharan dma_addr = dma_map_single(dev, skb->data, pkt_len, DMA_TO_DEVICE); 1093866b8b18SWingMan Kwok if (unlikely(dma_mapping_error(dev, dma_addr))) { 109484640e27SKaricheri, Muralidharan dev_err(netcp->ndev_dev, "Failed to map skb buffer\n"); 109584640e27SKaricheri, Muralidharan return NULL; 109684640e27SKaricheri, Muralidharan } 109784640e27SKaricheri, Muralidharan 109884640e27SKaricheri, Muralidharan desc = knav_pool_desc_get(netcp->tx_pool); 1099bf69a3b6SViresh Kumar if (IS_ERR_OR_NULL(desc)) { 110084640e27SKaricheri, Muralidharan dev_err(netcp->ndev_dev, "out of TX desc\n"); 110184640e27SKaricheri, Muralidharan dma_unmap_single(dev, dma_addr, pkt_len, DMA_TO_DEVICE); 110284640e27SKaricheri, Muralidharan return NULL; 110384640e27SKaricheri, Muralidharan } 110484640e27SKaricheri, Muralidharan 110584640e27SKaricheri, Muralidharan set_pkt_info(dma_addr, pkt_len, 0, desc); 110684640e27SKaricheri, Muralidharan if (skb_is_nonlinear(skb)) { 110784640e27SKaricheri, Muralidharan prefetchw(skb_shinfo(skb)); 110884640e27SKaricheri, Muralidharan } else { 110984640e27SKaricheri, Muralidharan desc->next_desc = 0; 111084640e27SKaricheri, Muralidharan goto upd_pkt_len; 111184640e27SKaricheri, Muralidharan } 111284640e27SKaricheri, Muralidharan 111384640e27SKaricheri, Muralidharan pdesc = desc; 111484640e27SKaricheri, Muralidharan 111584640e27SKaricheri, Muralidharan /* Handle the case where skb is fragmented in pages */ 111684640e27SKaricheri, Muralidharan for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 111784640e27SKaricheri, Muralidharan skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 111884640e27SKaricheri, Muralidharan struct page *page = skb_frag_page(frag); 111984640e27SKaricheri, Muralidharan u32 page_offset = frag->page_offset; 112084640e27SKaricheri, Muralidharan u32 buf_len = skb_frag_size(frag); 112184640e27SKaricheri, Muralidharan dma_addr_t desc_dma; 112289907779SArnd Bergmann u32 desc_dma_32; 112384640e27SKaricheri, Muralidharan 112484640e27SKaricheri, Muralidharan dma_addr = dma_map_page(dev, page, page_offset, buf_len, 112584640e27SKaricheri, Muralidharan DMA_TO_DEVICE); 112684640e27SKaricheri, Muralidharan if (unlikely(!dma_addr)) { 112784640e27SKaricheri, Muralidharan dev_err(netcp->ndev_dev, "Failed to map skb page\n"); 112884640e27SKaricheri, Muralidharan goto free_descs; 112984640e27SKaricheri, Muralidharan } 113084640e27SKaricheri, Muralidharan 113184640e27SKaricheri, Muralidharan ndesc = knav_pool_desc_get(netcp->tx_pool); 1132bf69a3b6SViresh Kumar if (IS_ERR_OR_NULL(ndesc)) { 113384640e27SKaricheri, Muralidharan dev_err(netcp->ndev_dev, "out of TX desc for frags\n"); 113484640e27SKaricheri, Muralidharan dma_unmap_page(dev, dma_addr, buf_len, DMA_TO_DEVICE); 113584640e27SKaricheri, Muralidharan goto free_descs; 113684640e27SKaricheri, Muralidharan } 113784640e27SKaricheri, Muralidharan 113889907779SArnd Bergmann desc_dma = knav_pool_desc_virt_to_dma(netcp->tx_pool, ndesc); 113984640e27SKaricheri, Muralidharan set_pkt_info(dma_addr, buf_len, 0, ndesc); 114089907779SArnd Bergmann desc_dma_32 = (u32)desc_dma; 114189907779SArnd Bergmann set_words(&desc_dma_32, 1, &pdesc->next_desc); 114284640e27SKaricheri, Muralidharan pkt_len += buf_len; 114384640e27SKaricheri, Muralidharan if (pdesc != desc) 114484640e27SKaricheri, Muralidharan knav_pool_desc_map(netcp->tx_pool, pdesc, 114584640e27SKaricheri, Muralidharan sizeof(*pdesc), &desc_dma, &dma_sz); 114684640e27SKaricheri, Muralidharan pdesc = ndesc; 114784640e27SKaricheri, Muralidharan } 114884640e27SKaricheri, Muralidharan if (pdesc != desc) 114984640e27SKaricheri, Muralidharan knav_pool_desc_map(netcp->tx_pool, pdesc, sizeof(*pdesc), 115084640e27SKaricheri, Muralidharan &dma_addr, &dma_sz); 115184640e27SKaricheri, Muralidharan 115284640e27SKaricheri, Muralidharan /* frag list based linkage is not supported for now. */ 115384640e27SKaricheri, Muralidharan if (skb_shinfo(skb)->frag_list) { 115484640e27SKaricheri, Muralidharan dev_err_ratelimited(netcp->ndev_dev, "NETIF_F_FRAGLIST not supported\n"); 115584640e27SKaricheri, Muralidharan goto free_descs; 115684640e27SKaricheri, Muralidharan } 115784640e27SKaricheri, Muralidharan 115884640e27SKaricheri, Muralidharan upd_pkt_len: 115984640e27SKaricheri, Muralidharan WARN_ON(pkt_len != skb->len); 116084640e27SKaricheri, Muralidharan 116184640e27SKaricheri, Muralidharan pkt_len &= KNAV_DMA_DESC_PKT_LEN_MASK; 116284640e27SKaricheri, Muralidharan set_words(&pkt_len, 1, &desc->desc_info); 116384640e27SKaricheri, Muralidharan return desc; 116484640e27SKaricheri, Muralidharan 116584640e27SKaricheri, Muralidharan free_descs: 116684640e27SKaricheri, Muralidharan netcp_free_tx_desc_chain(netcp, desc, sizeof(*desc)); 116784640e27SKaricheri, Muralidharan return NULL; 116884640e27SKaricheri, Muralidharan } 116984640e27SKaricheri, Muralidharan 117084640e27SKaricheri, Muralidharan static int netcp_tx_submit_skb(struct netcp_intf *netcp, 117184640e27SKaricheri, Muralidharan struct sk_buff *skb, 117284640e27SKaricheri, Muralidharan struct knav_dma_desc *desc) 117384640e27SKaricheri, Muralidharan { 117484640e27SKaricheri, Muralidharan struct netcp_tx_pipe *tx_pipe = NULL; 117584640e27SKaricheri, Muralidharan struct netcp_hook_list *tx_hook; 117684640e27SKaricheri, Muralidharan struct netcp_packet p_info; 11776246168bSWingMan Kwok struct netcp_tx_cb *tx_cb; 117884640e27SKaricheri, Muralidharan unsigned int dma_sz; 117984640e27SKaricheri, Muralidharan dma_addr_t dma; 1180e170f409SKaricheri, Muralidharan u32 tmp = 0; 118184640e27SKaricheri, Muralidharan int ret = 0; 118284640e27SKaricheri, Muralidharan 118384640e27SKaricheri, Muralidharan p_info.netcp = netcp; 118484640e27SKaricheri, Muralidharan p_info.skb = skb; 118584640e27SKaricheri, Muralidharan p_info.tx_pipe = NULL; 118684640e27SKaricheri, Muralidharan p_info.psdata_len = 0; 118784640e27SKaricheri, Muralidharan p_info.ts_context = NULL; 11886246168bSWingMan Kwok p_info.txtstamp = NULL; 118984640e27SKaricheri, Muralidharan p_info.epib = desc->epib; 11909dd2d6c5SArnd Bergmann p_info.psdata = (u32 __force *)desc->psdata; 11919dd2d6c5SArnd Bergmann memset(p_info.epib, 0, KNAV_DMA_NUM_EPIB_WORDS * sizeof(__le32)); 119284640e27SKaricheri, Muralidharan 119384640e27SKaricheri, Muralidharan /* Find out where to inject the packet for transmission */ 119484640e27SKaricheri, Muralidharan list_for_each_entry(tx_hook, &netcp->txhook_list_head, list) { 119584640e27SKaricheri, Muralidharan ret = tx_hook->hook_rtn(tx_hook->order, tx_hook->hook_data, 119684640e27SKaricheri, Muralidharan &p_info); 119784640e27SKaricheri, Muralidharan if (unlikely(ret != 0)) { 119884640e27SKaricheri, Muralidharan dev_err(netcp->ndev_dev, "TX hook %d rejected the packet with reason(%d)\n", 119984640e27SKaricheri, Muralidharan tx_hook->order, ret); 120084640e27SKaricheri, Muralidharan ret = (ret < 0) ? ret : NETDEV_TX_OK; 120184640e27SKaricheri, Muralidharan goto out; 120284640e27SKaricheri, Muralidharan } 120384640e27SKaricheri, Muralidharan } 120484640e27SKaricheri, Muralidharan 120584640e27SKaricheri, Muralidharan /* Make sure some TX hook claimed the packet */ 120684640e27SKaricheri, Muralidharan tx_pipe = p_info.tx_pipe; 120784640e27SKaricheri, Muralidharan if (!tx_pipe) { 120884640e27SKaricheri, Muralidharan dev_err(netcp->ndev_dev, "No TX hook claimed the packet!\n"); 120984640e27SKaricheri, Muralidharan ret = -ENXIO; 121084640e27SKaricheri, Muralidharan goto out; 121184640e27SKaricheri, Muralidharan } 121284640e27SKaricheri, Muralidharan 12136246168bSWingMan Kwok tx_cb = (struct netcp_tx_cb *)skb->cb; 12146246168bSWingMan Kwok tx_cb->ts_context = p_info.ts_context; 12156246168bSWingMan Kwok tx_cb->txtstamp = p_info.txtstamp; 12166246168bSWingMan Kwok 121784640e27SKaricheri, Muralidharan /* update descriptor */ 121884640e27SKaricheri, Muralidharan if (p_info.psdata_len) { 12199dd2d6c5SArnd Bergmann /* psdata points to both native-endian and device-endian data */ 12209dd2d6c5SArnd Bergmann __le32 *psdata = (void __force *)p_info.psdata; 122184640e27SKaricheri, Muralidharan 1222aa255101SKaricheri, Muralidharan set_words((u32 *)psdata + 1223aa255101SKaricheri, Muralidharan (KNAV_DMA_NUM_PS_WORDS - p_info.psdata_len), 1224aa255101SKaricheri, Muralidharan p_info.psdata_len, psdata); 1225e170f409SKaricheri, Muralidharan tmp |= (p_info.psdata_len & KNAV_DMA_DESC_PSLEN_MASK) << 122684640e27SKaricheri, Muralidharan KNAV_DMA_DESC_PSLEN_SHIFT; 122784640e27SKaricheri, Muralidharan } 122884640e27SKaricheri, Muralidharan 1229e170f409SKaricheri, Muralidharan tmp |= KNAV_DMA_DESC_HAS_EPIB | 123084640e27SKaricheri, Muralidharan ((netcp->tx_compl_qid & KNAV_DMA_DESC_RETQ_MASK) << 1231e170f409SKaricheri, Muralidharan KNAV_DMA_DESC_RETQ_SHIFT); 123284640e27SKaricheri, Muralidharan 1233e170f409SKaricheri, Muralidharan if (!(tx_pipe->flags & SWITCH_TO_PORT_IN_TAGINFO)) { 1234e170f409SKaricheri, Muralidharan tmp |= ((tx_pipe->switch_to_port & KNAV_DMA_DESC_PSFLAG_MASK) << 1235e170f409SKaricheri, Muralidharan KNAV_DMA_DESC_PSFLAG_SHIFT); 1236e170f409SKaricheri, Muralidharan } 1237e170f409SKaricheri, Muralidharan 1238e170f409SKaricheri, Muralidharan set_words(&tmp, 1, &desc->packet_info); 123906324481SKaricheri, Muralidharan /* warning!!!! We are saving the virtual ptr in the sw_data 124006324481SKaricheri, Muralidharan * field as a 32bit value. Will not work on 64bit machines 124106324481SKaricheri, Muralidharan */ 124206324481SKaricheri, Muralidharan SET_SW_DATA0((u32)skb, desc); 124384640e27SKaricheri, Muralidharan 1244e170f409SKaricheri, Muralidharan if (tx_pipe->flags & SWITCH_TO_PORT_IN_TAGINFO) { 1245e170f409SKaricheri, Muralidharan tmp = tx_pipe->switch_to_port; 124689907779SArnd Bergmann set_words(&tmp, 1, &desc->tag_info); 1247e170f409SKaricheri, Muralidharan } 1248e170f409SKaricheri, Muralidharan 124984640e27SKaricheri, Muralidharan /* submit packet descriptor */ 125084640e27SKaricheri, Muralidharan ret = knav_pool_desc_map(netcp->tx_pool, desc, sizeof(*desc), &dma, 125184640e27SKaricheri, Muralidharan &dma_sz); 125284640e27SKaricheri, Muralidharan if (unlikely(ret)) { 125384640e27SKaricheri, Muralidharan dev_err(netcp->ndev_dev, "%s() failed to map desc\n", __func__); 125484640e27SKaricheri, Muralidharan ret = -ENOMEM; 125584640e27SKaricheri, Muralidharan goto out; 125684640e27SKaricheri, Muralidharan } 125784640e27SKaricheri, Muralidharan skb_tx_timestamp(skb); 125884640e27SKaricheri, Muralidharan knav_queue_push(tx_pipe->dma_queue, dma, dma_sz, 0); 125984640e27SKaricheri, Muralidharan 126084640e27SKaricheri, Muralidharan out: 126184640e27SKaricheri, Muralidharan return ret; 126284640e27SKaricheri, Muralidharan } 126384640e27SKaricheri, Muralidharan 126484640e27SKaricheri, Muralidharan /* Submit the packet */ 126584640e27SKaricheri, Muralidharan static int netcp_ndo_start_xmit(struct sk_buff *skb, struct net_device *ndev) 126684640e27SKaricheri, Muralidharan { 126784640e27SKaricheri, Muralidharan struct netcp_intf *netcp = netdev_priv(ndev); 12686a8162e9SMichael Scherban struct netcp_stats *tx_stats = &netcp->stats; 126984640e27SKaricheri, Muralidharan int subqueue = skb_get_queue_mapping(skb); 127084640e27SKaricheri, Muralidharan struct knav_dma_desc *desc; 127184640e27SKaricheri, Muralidharan int desc_count, ret = 0; 127284640e27SKaricheri, Muralidharan 127384640e27SKaricheri, Muralidharan if (unlikely(skb->len <= 0)) { 127484640e27SKaricheri, Muralidharan dev_kfree_skb(skb); 127584640e27SKaricheri, Muralidharan return NETDEV_TX_OK; 127684640e27SKaricheri, Muralidharan } 127784640e27SKaricheri, Muralidharan 127884640e27SKaricheri, Muralidharan if (unlikely(skb->len < NETCP_MIN_PACKET_SIZE)) { 127984640e27SKaricheri, Muralidharan ret = skb_padto(skb, NETCP_MIN_PACKET_SIZE); 128084640e27SKaricheri, Muralidharan if (ret < 0) { 128184640e27SKaricheri, Muralidharan /* If we get here, the skb has already been dropped */ 128284640e27SKaricheri, Muralidharan dev_warn(netcp->ndev_dev, "padding failed (%d), packet dropped\n", 128384640e27SKaricheri, Muralidharan ret); 12846a8162e9SMichael Scherban tx_stats->tx_dropped++; 128584640e27SKaricheri, Muralidharan return ret; 128684640e27SKaricheri, Muralidharan } 128784640e27SKaricheri, Muralidharan skb->len = NETCP_MIN_PACKET_SIZE; 128884640e27SKaricheri, Muralidharan } 128984640e27SKaricheri, Muralidharan 129084640e27SKaricheri, Muralidharan desc = netcp_tx_map_skb(skb, netcp); 129184640e27SKaricheri, Muralidharan if (unlikely(!desc)) { 129284640e27SKaricheri, Muralidharan netif_stop_subqueue(ndev, subqueue); 129384640e27SKaricheri, Muralidharan ret = -ENOBUFS; 129484640e27SKaricheri, Muralidharan goto drop; 129584640e27SKaricheri, Muralidharan } 129684640e27SKaricheri, Muralidharan 129784640e27SKaricheri, Muralidharan ret = netcp_tx_submit_skb(netcp, skb, desc); 129884640e27SKaricheri, Muralidharan if (ret) 129984640e27SKaricheri, Muralidharan goto drop; 130084640e27SKaricheri, Muralidharan 130184640e27SKaricheri, Muralidharan /* Check Tx pool count & stop subqueue if needed */ 130284640e27SKaricheri, Muralidharan desc_count = knav_pool_count(netcp->tx_pool); 130384640e27SKaricheri, Muralidharan if (desc_count < netcp->tx_pause_threshold) { 130484640e27SKaricheri, Muralidharan dev_dbg(netcp->ndev_dev, "pausing tx, count(%d)\n", desc_count); 130584640e27SKaricheri, Muralidharan netif_stop_subqueue(ndev, subqueue); 130684640e27SKaricheri, Muralidharan } 130784640e27SKaricheri, Muralidharan return NETDEV_TX_OK; 130884640e27SKaricheri, Muralidharan 130984640e27SKaricheri, Muralidharan drop: 13106a8162e9SMichael Scherban tx_stats->tx_dropped++; 131184640e27SKaricheri, Muralidharan if (desc) 131284640e27SKaricheri, Muralidharan netcp_free_tx_desc_chain(netcp, desc, sizeof(*desc)); 131384640e27SKaricheri, Muralidharan dev_kfree_skb(skb); 131484640e27SKaricheri, Muralidharan return ret; 131584640e27SKaricheri, Muralidharan } 131684640e27SKaricheri, Muralidharan 131784640e27SKaricheri, Muralidharan int netcp_txpipe_close(struct netcp_tx_pipe *tx_pipe) 131884640e27SKaricheri, Muralidharan { 131984640e27SKaricheri, Muralidharan if (tx_pipe->dma_channel) { 132084640e27SKaricheri, Muralidharan knav_dma_close_channel(tx_pipe->dma_channel); 132184640e27SKaricheri, Muralidharan tx_pipe->dma_channel = NULL; 132284640e27SKaricheri, Muralidharan } 132384640e27SKaricheri, Muralidharan return 0; 132484640e27SKaricheri, Muralidharan } 132558c11b5fSKaricheri, Muralidharan EXPORT_SYMBOL_GPL(netcp_txpipe_close); 132684640e27SKaricheri, Muralidharan 132784640e27SKaricheri, Muralidharan int netcp_txpipe_open(struct netcp_tx_pipe *tx_pipe) 132884640e27SKaricheri, Muralidharan { 132984640e27SKaricheri, Muralidharan struct device *dev = tx_pipe->netcp_device->device; 133084640e27SKaricheri, Muralidharan struct knav_dma_cfg config; 133184640e27SKaricheri, Muralidharan int ret = 0; 133284640e27SKaricheri, Muralidharan u8 name[16]; 133384640e27SKaricheri, Muralidharan 133484640e27SKaricheri, Muralidharan memset(&config, 0, sizeof(config)); 133584640e27SKaricheri, Muralidharan config.direction = DMA_MEM_TO_DEV; 133684640e27SKaricheri, Muralidharan config.u.tx.filt_einfo = false; 133784640e27SKaricheri, Muralidharan config.u.tx.filt_pswords = false; 133884640e27SKaricheri, Muralidharan config.u.tx.priority = DMA_PRIO_MED_L; 133984640e27SKaricheri, Muralidharan 134084640e27SKaricheri, Muralidharan tx_pipe->dma_channel = knav_dma_open_channel(dev, 134184640e27SKaricheri, Muralidharan tx_pipe->dma_chan_name, &config); 13425b6cb43bSIvan Khoronzhuk if (IS_ERR(tx_pipe->dma_channel)) { 134384640e27SKaricheri, Muralidharan dev_err(dev, "failed opening tx chan(%s)\n", 134484640e27SKaricheri, Muralidharan tx_pipe->dma_chan_name); 13455b6cb43bSIvan Khoronzhuk ret = PTR_ERR(tx_pipe->dma_channel); 134684640e27SKaricheri, Muralidharan goto err; 134784640e27SKaricheri, Muralidharan } 134884640e27SKaricheri, Muralidharan 134984640e27SKaricheri, Muralidharan snprintf(name, sizeof(name), "tx-pipe-%s", dev_name(dev)); 135084640e27SKaricheri, Muralidharan tx_pipe->dma_queue = knav_queue_open(name, tx_pipe->dma_queue_id, 135184640e27SKaricheri, Muralidharan KNAV_QUEUE_SHARED); 135284640e27SKaricheri, Muralidharan if (IS_ERR(tx_pipe->dma_queue)) { 135384640e27SKaricheri, Muralidharan dev_err(dev, "Could not open DMA queue for channel \"%s\": %d\n", 135484640e27SKaricheri, Muralidharan name, ret); 135584640e27SKaricheri, Muralidharan ret = PTR_ERR(tx_pipe->dma_queue); 135684640e27SKaricheri, Muralidharan goto err; 135784640e27SKaricheri, Muralidharan } 135884640e27SKaricheri, Muralidharan 135984640e27SKaricheri, Muralidharan dev_dbg(dev, "opened tx pipe %s\n", name); 136084640e27SKaricheri, Muralidharan return 0; 136184640e27SKaricheri, Muralidharan 136284640e27SKaricheri, Muralidharan err: 136384640e27SKaricheri, Muralidharan if (!IS_ERR_OR_NULL(tx_pipe->dma_channel)) 136484640e27SKaricheri, Muralidharan knav_dma_close_channel(tx_pipe->dma_channel); 136584640e27SKaricheri, Muralidharan tx_pipe->dma_channel = NULL; 136684640e27SKaricheri, Muralidharan return ret; 136784640e27SKaricheri, Muralidharan } 136858c11b5fSKaricheri, Muralidharan EXPORT_SYMBOL_GPL(netcp_txpipe_open); 136984640e27SKaricheri, Muralidharan 137084640e27SKaricheri, Muralidharan int netcp_txpipe_init(struct netcp_tx_pipe *tx_pipe, 137184640e27SKaricheri, Muralidharan struct netcp_device *netcp_device, 137284640e27SKaricheri, Muralidharan const char *dma_chan_name, unsigned int dma_queue_id) 137384640e27SKaricheri, Muralidharan { 137484640e27SKaricheri, Muralidharan memset(tx_pipe, 0, sizeof(*tx_pipe)); 137584640e27SKaricheri, Muralidharan tx_pipe->netcp_device = netcp_device; 137684640e27SKaricheri, Muralidharan tx_pipe->dma_chan_name = dma_chan_name; 137784640e27SKaricheri, Muralidharan tx_pipe->dma_queue_id = dma_queue_id; 137884640e27SKaricheri, Muralidharan return 0; 137984640e27SKaricheri, Muralidharan } 138058c11b5fSKaricheri, Muralidharan EXPORT_SYMBOL_GPL(netcp_txpipe_init); 138184640e27SKaricheri, Muralidharan 138284640e27SKaricheri, Muralidharan static struct netcp_addr *netcp_addr_find(struct netcp_intf *netcp, 138384640e27SKaricheri, Muralidharan const u8 *addr, 138484640e27SKaricheri, Muralidharan enum netcp_addr_type type) 138584640e27SKaricheri, Muralidharan { 138684640e27SKaricheri, Muralidharan struct netcp_addr *naddr; 138784640e27SKaricheri, Muralidharan 138884640e27SKaricheri, Muralidharan list_for_each_entry(naddr, &netcp->addr_list, node) { 138984640e27SKaricheri, Muralidharan if (naddr->type != type) 139084640e27SKaricheri, Muralidharan continue; 139184640e27SKaricheri, Muralidharan if (addr && memcmp(addr, naddr->addr, ETH_ALEN)) 139284640e27SKaricheri, Muralidharan continue; 139384640e27SKaricheri, Muralidharan return naddr; 139484640e27SKaricheri, Muralidharan } 139584640e27SKaricheri, Muralidharan 139684640e27SKaricheri, Muralidharan return NULL; 139784640e27SKaricheri, Muralidharan } 139884640e27SKaricheri, Muralidharan 139984640e27SKaricheri, Muralidharan static struct netcp_addr *netcp_addr_add(struct netcp_intf *netcp, 140084640e27SKaricheri, Muralidharan const u8 *addr, 140184640e27SKaricheri, Muralidharan enum netcp_addr_type type) 140284640e27SKaricheri, Muralidharan { 140384640e27SKaricheri, Muralidharan struct netcp_addr *naddr; 140484640e27SKaricheri, Muralidharan 140584640e27SKaricheri, Muralidharan naddr = devm_kmalloc(netcp->dev, sizeof(*naddr), GFP_ATOMIC); 140684640e27SKaricheri, Muralidharan if (!naddr) 140784640e27SKaricheri, Muralidharan return NULL; 140884640e27SKaricheri, Muralidharan 140984640e27SKaricheri, Muralidharan naddr->type = type; 141084640e27SKaricheri, Muralidharan naddr->flags = 0; 141184640e27SKaricheri, Muralidharan naddr->netcp = netcp; 141284640e27SKaricheri, Muralidharan if (addr) 141384640e27SKaricheri, Muralidharan ether_addr_copy(naddr->addr, addr); 141484640e27SKaricheri, Muralidharan else 1415c7bf7169SJoe Perches eth_zero_addr(naddr->addr); 141684640e27SKaricheri, Muralidharan list_add_tail(&naddr->node, &netcp->addr_list); 141784640e27SKaricheri, Muralidharan 141884640e27SKaricheri, Muralidharan return naddr; 141984640e27SKaricheri, Muralidharan } 142084640e27SKaricheri, Muralidharan 142184640e27SKaricheri, Muralidharan static void netcp_addr_del(struct netcp_intf *netcp, struct netcp_addr *naddr) 142284640e27SKaricheri, Muralidharan { 142384640e27SKaricheri, Muralidharan list_del(&naddr->node); 142484640e27SKaricheri, Muralidharan devm_kfree(netcp->dev, naddr); 142584640e27SKaricheri, Muralidharan } 142684640e27SKaricheri, Muralidharan 142784640e27SKaricheri, Muralidharan static void netcp_addr_clear_mark(struct netcp_intf *netcp) 142884640e27SKaricheri, Muralidharan { 142984640e27SKaricheri, Muralidharan struct netcp_addr *naddr; 143084640e27SKaricheri, Muralidharan 143184640e27SKaricheri, Muralidharan list_for_each_entry(naddr, &netcp->addr_list, node) 143284640e27SKaricheri, Muralidharan naddr->flags = 0; 143384640e27SKaricheri, Muralidharan } 143484640e27SKaricheri, Muralidharan 143584640e27SKaricheri, Muralidharan static void netcp_addr_add_mark(struct netcp_intf *netcp, const u8 *addr, 143684640e27SKaricheri, Muralidharan enum netcp_addr_type type) 143784640e27SKaricheri, Muralidharan { 143884640e27SKaricheri, Muralidharan struct netcp_addr *naddr; 143984640e27SKaricheri, Muralidharan 144084640e27SKaricheri, Muralidharan naddr = netcp_addr_find(netcp, addr, type); 144184640e27SKaricheri, Muralidharan if (naddr) { 144284640e27SKaricheri, Muralidharan naddr->flags |= ADDR_VALID; 144384640e27SKaricheri, Muralidharan return; 144484640e27SKaricheri, Muralidharan } 144584640e27SKaricheri, Muralidharan 144684640e27SKaricheri, Muralidharan naddr = netcp_addr_add(netcp, addr, type); 144784640e27SKaricheri, Muralidharan if (!WARN_ON(!naddr)) 144884640e27SKaricheri, Muralidharan naddr->flags |= ADDR_NEW; 144984640e27SKaricheri, Muralidharan } 145084640e27SKaricheri, Muralidharan 145184640e27SKaricheri, Muralidharan static void netcp_addr_sweep_del(struct netcp_intf *netcp) 145284640e27SKaricheri, Muralidharan { 145384640e27SKaricheri, Muralidharan struct netcp_addr *naddr, *tmp; 145484640e27SKaricheri, Muralidharan struct netcp_intf_modpriv *priv; 145584640e27SKaricheri, Muralidharan struct netcp_module *module; 145684640e27SKaricheri, Muralidharan int error; 145784640e27SKaricheri, Muralidharan 145884640e27SKaricheri, Muralidharan list_for_each_entry_safe(naddr, tmp, &netcp->addr_list, node) { 145984640e27SKaricheri, Muralidharan if (naddr->flags & (ADDR_VALID | ADDR_NEW)) 146084640e27SKaricheri, Muralidharan continue; 146184640e27SKaricheri, Muralidharan dev_dbg(netcp->ndev_dev, "deleting address %pM, type %x\n", 146284640e27SKaricheri, Muralidharan naddr->addr, naddr->type); 146384640e27SKaricheri, Muralidharan for_each_module(netcp, priv) { 146484640e27SKaricheri, Muralidharan module = priv->netcp_module; 146584640e27SKaricheri, Muralidharan if (!module->del_addr) 146684640e27SKaricheri, Muralidharan continue; 146784640e27SKaricheri, Muralidharan error = module->del_addr(priv->module_priv, 146884640e27SKaricheri, Muralidharan naddr); 146984640e27SKaricheri, Muralidharan WARN_ON(error); 147084640e27SKaricheri, Muralidharan } 147184640e27SKaricheri, Muralidharan netcp_addr_del(netcp, naddr); 147284640e27SKaricheri, Muralidharan } 147384640e27SKaricheri, Muralidharan } 147484640e27SKaricheri, Muralidharan 147584640e27SKaricheri, Muralidharan static void netcp_addr_sweep_add(struct netcp_intf *netcp) 147684640e27SKaricheri, Muralidharan { 147784640e27SKaricheri, Muralidharan struct netcp_addr *naddr, *tmp; 147884640e27SKaricheri, Muralidharan struct netcp_intf_modpriv *priv; 147984640e27SKaricheri, Muralidharan struct netcp_module *module; 148084640e27SKaricheri, Muralidharan int error; 148184640e27SKaricheri, Muralidharan 148284640e27SKaricheri, Muralidharan list_for_each_entry_safe(naddr, tmp, &netcp->addr_list, node) { 148384640e27SKaricheri, Muralidharan if (!(naddr->flags & ADDR_NEW)) 148484640e27SKaricheri, Muralidharan continue; 148584640e27SKaricheri, Muralidharan dev_dbg(netcp->ndev_dev, "adding address %pM, type %x\n", 148684640e27SKaricheri, Muralidharan naddr->addr, naddr->type); 14878ceaf361SKaricheri, Muralidharan 148884640e27SKaricheri, Muralidharan for_each_module(netcp, priv) { 148984640e27SKaricheri, Muralidharan module = priv->netcp_module; 149084640e27SKaricheri, Muralidharan if (!module->add_addr) 149184640e27SKaricheri, Muralidharan continue; 149284640e27SKaricheri, Muralidharan error = module->add_addr(priv->module_priv, naddr); 149384640e27SKaricheri, Muralidharan WARN_ON(error); 149484640e27SKaricheri, Muralidharan } 149584640e27SKaricheri, Muralidharan } 149684640e27SKaricheri, Muralidharan } 149784640e27SKaricheri, Muralidharan 14980542a87cSWingMan Kwok static int netcp_set_promiscuous(struct netcp_intf *netcp, bool promisc) 14990542a87cSWingMan Kwok { 15000542a87cSWingMan Kwok struct netcp_intf_modpriv *priv; 15010542a87cSWingMan Kwok struct netcp_module *module; 15020542a87cSWingMan Kwok int error; 15030542a87cSWingMan Kwok 15040542a87cSWingMan Kwok for_each_module(netcp, priv) { 15050542a87cSWingMan Kwok module = priv->netcp_module; 15060542a87cSWingMan Kwok if (!module->set_rx_mode) 15070542a87cSWingMan Kwok continue; 15080542a87cSWingMan Kwok 15090542a87cSWingMan Kwok error = module->set_rx_mode(priv->module_priv, promisc); 15100542a87cSWingMan Kwok if (error) 15110542a87cSWingMan Kwok return error; 15120542a87cSWingMan Kwok } 15130542a87cSWingMan Kwok return 0; 15140542a87cSWingMan Kwok } 15150542a87cSWingMan Kwok 151684640e27SKaricheri, Muralidharan static void netcp_set_rx_mode(struct net_device *ndev) 151784640e27SKaricheri, Muralidharan { 151884640e27SKaricheri, Muralidharan struct netcp_intf *netcp = netdev_priv(ndev); 151984640e27SKaricheri, Muralidharan struct netdev_hw_addr *ndev_addr; 152084640e27SKaricheri, Muralidharan bool promisc; 152184640e27SKaricheri, Muralidharan 152284640e27SKaricheri, Muralidharan promisc = (ndev->flags & IFF_PROMISC || 152384640e27SKaricheri, Muralidharan ndev->flags & IFF_ALLMULTI || 152484640e27SKaricheri, Muralidharan netdev_mc_count(ndev) > NETCP_MAX_MCAST_ADDR); 152584640e27SKaricheri, Muralidharan 15268ceaf361SKaricheri, Muralidharan spin_lock(&netcp->lock); 152784640e27SKaricheri, Muralidharan /* first clear all marks */ 152884640e27SKaricheri, Muralidharan netcp_addr_clear_mark(netcp); 152984640e27SKaricheri, Muralidharan 153084640e27SKaricheri, Muralidharan /* next add new entries, mark existing ones */ 153184640e27SKaricheri, Muralidharan netcp_addr_add_mark(netcp, ndev->broadcast, ADDR_BCAST); 153284640e27SKaricheri, Muralidharan for_each_dev_addr(ndev, ndev_addr) 153384640e27SKaricheri, Muralidharan netcp_addr_add_mark(netcp, ndev_addr->addr, ADDR_DEV); 153484640e27SKaricheri, Muralidharan netdev_for_each_uc_addr(ndev_addr, ndev) 153584640e27SKaricheri, Muralidharan netcp_addr_add_mark(netcp, ndev_addr->addr, ADDR_UCAST); 153684640e27SKaricheri, Muralidharan netdev_for_each_mc_addr(ndev_addr, ndev) 153784640e27SKaricheri, Muralidharan netcp_addr_add_mark(netcp, ndev_addr->addr, ADDR_MCAST); 153884640e27SKaricheri, Muralidharan 153984640e27SKaricheri, Muralidharan if (promisc) 154084640e27SKaricheri, Muralidharan netcp_addr_add_mark(netcp, NULL, ADDR_ANY); 154184640e27SKaricheri, Muralidharan 154284640e27SKaricheri, Muralidharan /* finally sweep and callout into modules */ 154384640e27SKaricheri, Muralidharan netcp_addr_sweep_del(netcp); 154484640e27SKaricheri, Muralidharan netcp_addr_sweep_add(netcp); 15450542a87cSWingMan Kwok netcp_set_promiscuous(netcp, promisc); 15468ceaf361SKaricheri, Muralidharan spin_unlock(&netcp->lock); 154784640e27SKaricheri, Muralidharan } 154884640e27SKaricheri, Muralidharan 154984640e27SKaricheri, Muralidharan static void netcp_free_navigator_resources(struct netcp_intf *netcp) 155084640e27SKaricheri, Muralidharan { 155184640e27SKaricheri, Muralidharan int i; 155284640e27SKaricheri, Muralidharan 155384640e27SKaricheri, Muralidharan if (netcp->rx_channel) { 155484640e27SKaricheri, Muralidharan knav_dma_close_channel(netcp->rx_channel); 155584640e27SKaricheri, Muralidharan netcp->rx_channel = NULL; 155684640e27SKaricheri, Muralidharan } 155784640e27SKaricheri, Muralidharan 155884640e27SKaricheri, Muralidharan if (!IS_ERR_OR_NULL(netcp->rx_pool)) 155984640e27SKaricheri, Muralidharan netcp_rxpool_free(netcp); 156084640e27SKaricheri, Muralidharan 156184640e27SKaricheri, Muralidharan if (!IS_ERR_OR_NULL(netcp->rx_queue)) { 156284640e27SKaricheri, Muralidharan knav_queue_close(netcp->rx_queue); 156384640e27SKaricheri, Muralidharan netcp->rx_queue = NULL; 156484640e27SKaricheri, Muralidharan } 156584640e27SKaricheri, Muralidharan 156684640e27SKaricheri, Muralidharan for (i = 0; i < KNAV_DMA_FDQ_PER_CHAN && 156784640e27SKaricheri, Muralidharan !IS_ERR_OR_NULL(netcp->rx_fdq[i]) ; ++i) { 156884640e27SKaricheri, Muralidharan knav_queue_close(netcp->rx_fdq[i]); 156984640e27SKaricheri, Muralidharan netcp->rx_fdq[i] = NULL; 157084640e27SKaricheri, Muralidharan } 157184640e27SKaricheri, Muralidharan 157284640e27SKaricheri, Muralidharan if (!IS_ERR_OR_NULL(netcp->tx_compl_q)) { 157384640e27SKaricheri, Muralidharan knav_queue_close(netcp->tx_compl_q); 157484640e27SKaricheri, Muralidharan netcp->tx_compl_q = NULL; 157584640e27SKaricheri, Muralidharan } 157684640e27SKaricheri, Muralidharan 157784640e27SKaricheri, Muralidharan if (!IS_ERR_OR_NULL(netcp->tx_pool)) { 157884640e27SKaricheri, Muralidharan knav_pool_destroy(netcp->tx_pool); 157984640e27SKaricheri, Muralidharan netcp->tx_pool = NULL; 158084640e27SKaricheri, Muralidharan } 158184640e27SKaricheri, Muralidharan } 158284640e27SKaricheri, Muralidharan 158384640e27SKaricheri, Muralidharan static int netcp_setup_navigator_resources(struct net_device *ndev) 158484640e27SKaricheri, Muralidharan { 158584640e27SKaricheri, Muralidharan struct netcp_intf *netcp = netdev_priv(ndev); 158684640e27SKaricheri, Muralidharan struct knav_queue_notify_config notify_cfg; 158784640e27SKaricheri, Muralidharan struct knav_dma_cfg config; 158884640e27SKaricheri, Muralidharan u32 last_fdq = 0; 158984640e27SKaricheri, Muralidharan u8 name[16]; 159084640e27SKaricheri, Muralidharan int ret; 159184640e27SKaricheri, Muralidharan int i; 159284640e27SKaricheri, Muralidharan 159384640e27SKaricheri, Muralidharan /* Create Rx/Tx descriptor pools */ 159484640e27SKaricheri, Muralidharan snprintf(name, sizeof(name), "rx-pool-%s", ndev->name); 159584640e27SKaricheri, Muralidharan netcp->rx_pool = knav_pool_create(name, netcp->rx_pool_size, 159684640e27SKaricheri, Muralidharan netcp->rx_pool_region_id); 159784640e27SKaricheri, Muralidharan if (IS_ERR_OR_NULL(netcp->rx_pool)) { 159884640e27SKaricheri, Muralidharan dev_err(netcp->ndev_dev, "Couldn't create rx pool\n"); 159984640e27SKaricheri, Muralidharan ret = PTR_ERR(netcp->rx_pool); 160084640e27SKaricheri, Muralidharan goto fail; 160184640e27SKaricheri, Muralidharan } 160284640e27SKaricheri, Muralidharan 160384640e27SKaricheri, Muralidharan snprintf(name, sizeof(name), "tx-pool-%s", ndev->name); 160484640e27SKaricheri, Muralidharan netcp->tx_pool = knav_pool_create(name, netcp->tx_pool_size, 160584640e27SKaricheri, Muralidharan netcp->tx_pool_region_id); 160684640e27SKaricheri, Muralidharan if (IS_ERR_OR_NULL(netcp->tx_pool)) { 160784640e27SKaricheri, Muralidharan dev_err(netcp->ndev_dev, "Couldn't create tx pool\n"); 160884640e27SKaricheri, Muralidharan ret = PTR_ERR(netcp->tx_pool); 160984640e27SKaricheri, Muralidharan goto fail; 161084640e27SKaricheri, Muralidharan } 161184640e27SKaricheri, Muralidharan 161284640e27SKaricheri, Muralidharan /* open Tx completion queue */ 161384640e27SKaricheri, Muralidharan snprintf(name, sizeof(name), "tx-compl-%s", ndev->name); 161484640e27SKaricheri, Muralidharan netcp->tx_compl_q = knav_queue_open(name, netcp->tx_compl_qid, 0); 1615d01eb808SJulia Lawall if (IS_ERR(netcp->tx_compl_q)) { 161684640e27SKaricheri, Muralidharan ret = PTR_ERR(netcp->tx_compl_q); 161784640e27SKaricheri, Muralidharan goto fail; 161884640e27SKaricheri, Muralidharan } 161984640e27SKaricheri, Muralidharan netcp->tx_compl_qid = knav_queue_get_id(netcp->tx_compl_q); 162084640e27SKaricheri, Muralidharan 162184640e27SKaricheri, Muralidharan /* Set notification for Tx completion */ 162284640e27SKaricheri, Muralidharan notify_cfg.fn = netcp_tx_notify; 162384640e27SKaricheri, Muralidharan notify_cfg.fn_arg = netcp; 162484640e27SKaricheri, Muralidharan ret = knav_queue_device_control(netcp->tx_compl_q, 162584640e27SKaricheri, Muralidharan KNAV_QUEUE_SET_NOTIFIER, 162684640e27SKaricheri, Muralidharan (unsigned long)¬ify_cfg); 162784640e27SKaricheri, Muralidharan if (ret) 162884640e27SKaricheri, Muralidharan goto fail; 162984640e27SKaricheri, Muralidharan 163084640e27SKaricheri, Muralidharan knav_queue_disable_notify(netcp->tx_compl_q); 163184640e27SKaricheri, Muralidharan 163284640e27SKaricheri, Muralidharan /* open Rx completion queue */ 163384640e27SKaricheri, Muralidharan snprintf(name, sizeof(name), "rx-compl-%s", ndev->name); 163484640e27SKaricheri, Muralidharan netcp->rx_queue = knav_queue_open(name, netcp->rx_queue_id, 0); 1635d01eb808SJulia Lawall if (IS_ERR(netcp->rx_queue)) { 163684640e27SKaricheri, Muralidharan ret = PTR_ERR(netcp->rx_queue); 163784640e27SKaricheri, Muralidharan goto fail; 163884640e27SKaricheri, Muralidharan } 163984640e27SKaricheri, Muralidharan netcp->rx_queue_id = knav_queue_get_id(netcp->rx_queue); 164084640e27SKaricheri, Muralidharan 164184640e27SKaricheri, Muralidharan /* Set notification for Rx completion */ 164284640e27SKaricheri, Muralidharan notify_cfg.fn = netcp_rx_notify; 164384640e27SKaricheri, Muralidharan notify_cfg.fn_arg = netcp; 164484640e27SKaricheri, Muralidharan ret = knav_queue_device_control(netcp->rx_queue, 164584640e27SKaricheri, Muralidharan KNAV_QUEUE_SET_NOTIFIER, 164684640e27SKaricheri, Muralidharan (unsigned long)¬ify_cfg); 164784640e27SKaricheri, Muralidharan if (ret) 164884640e27SKaricheri, Muralidharan goto fail; 164984640e27SKaricheri, Muralidharan 165084640e27SKaricheri, Muralidharan knav_queue_disable_notify(netcp->rx_queue); 165184640e27SKaricheri, Muralidharan 165284640e27SKaricheri, Muralidharan /* open Rx FDQs */ 1653866b8b18SWingMan Kwok for (i = 0; i < KNAV_DMA_FDQ_PER_CHAN && netcp->rx_queue_depths[i]; 1654866b8b18SWingMan Kwok ++i) { 165584640e27SKaricheri, Muralidharan snprintf(name, sizeof(name), "rx-fdq-%s-%d", ndev->name, i); 165684640e27SKaricheri, Muralidharan netcp->rx_fdq[i] = knav_queue_open(name, KNAV_QUEUE_GP, 0); 1657d01eb808SJulia Lawall if (IS_ERR(netcp->rx_fdq[i])) { 165884640e27SKaricheri, Muralidharan ret = PTR_ERR(netcp->rx_fdq[i]); 165984640e27SKaricheri, Muralidharan goto fail; 166084640e27SKaricheri, Muralidharan } 166184640e27SKaricheri, Muralidharan } 166284640e27SKaricheri, Muralidharan 166384640e27SKaricheri, Muralidharan memset(&config, 0, sizeof(config)); 166484640e27SKaricheri, Muralidharan config.direction = DMA_DEV_TO_MEM; 166584640e27SKaricheri, Muralidharan config.u.rx.einfo_present = true; 166684640e27SKaricheri, Muralidharan config.u.rx.psinfo_present = true; 166784640e27SKaricheri, Muralidharan config.u.rx.err_mode = DMA_DROP; 166884640e27SKaricheri, Muralidharan config.u.rx.desc_type = DMA_DESC_HOST; 166984640e27SKaricheri, Muralidharan config.u.rx.psinfo_at_sop = false; 167084640e27SKaricheri, Muralidharan config.u.rx.sop_offset = NETCP_SOP_OFFSET; 167184640e27SKaricheri, Muralidharan config.u.rx.dst_q = netcp->rx_queue_id; 167284640e27SKaricheri, Muralidharan config.u.rx.thresh = DMA_THRESH_NONE; 167384640e27SKaricheri, Muralidharan 167484640e27SKaricheri, Muralidharan for (i = 0; i < KNAV_DMA_FDQ_PER_CHAN; ++i) { 167584640e27SKaricheri, Muralidharan if (netcp->rx_fdq[i]) 167684640e27SKaricheri, Muralidharan last_fdq = knav_queue_get_id(netcp->rx_fdq[i]); 167784640e27SKaricheri, Muralidharan config.u.rx.fdq[i] = last_fdq; 167884640e27SKaricheri, Muralidharan } 167984640e27SKaricheri, Muralidharan 168084640e27SKaricheri, Muralidharan netcp->rx_channel = knav_dma_open_channel(netcp->netcp_device->device, 168184640e27SKaricheri, Muralidharan netcp->dma_chan_name, &config); 16825b6cb43bSIvan Khoronzhuk if (IS_ERR(netcp->rx_channel)) { 168384640e27SKaricheri, Muralidharan dev_err(netcp->ndev_dev, "failed opening rx chan(%s\n", 168484640e27SKaricheri, Muralidharan netcp->dma_chan_name); 16855b6cb43bSIvan Khoronzhuk ret = PTR_ERR(netcp->rx_channel); 168684640e27SKaricheri, Muralidharan goto fail; 168784640e27SKaricheri, Muralidharan } 168884640e27SKaricheri, Muralidharan 168984640e27SKaricheri, Muralidharan dev_dbg(netcp->ndev_dev, "opened RX channel: %p\n", netcp->rx_channel); 169084640e27SKaricheri, Muralidharan return 0; 169184640e27SKaricheri, Muralidharan 169284640e27SKaricheri, Muralidharan fail: 169384640e27SKaricheri, Muralidharan netcp_free_navigator_resources(netcp); 169484640e27SKaricheri, Muralidharan return ret; 169584640e27SKaricheri, Muralidharan } 169684640e27SKaricheri, Muralidharan 169784640e27SKaricheri, Muralidharan /* Open the device */ 169884640e27SKaricheri, Muralidharan static int netcp_ndo_open(struct net_device *ndev) 169984640e27SKaricheri, Muralidharan { 170084640e27SKaricheri, Muralidharan struct netcp_intf *netcp = netdev_priv(ndev); 170184640e27SKaricheri, Muralidharan struct netcp_intf_modpriv *intf_modpriv; 170284640e27SKaricheri, Muralidharan struct netcp_module *module; 170384640e27SKaricheri, Muralidharan int ret; 170484640e27SKaricheri, Muralidharan 170584640e27SKaricheri, Muralidharan netif_carrier_off(ndev); 170684640e27SKaricheri, Muralidharan ret = netcp_setup_navigator_resources(ndev); 170784640e27SKaricheri, Muralidharan if (ret) { 170884640e27SKaricheri, Muralidharan dev_err(netcp->ndev_dev, "Failed to setup navigator resources\n"); 170984640e27SKaricheri, Muralidharan goto fail; 171084640e27SKaricheri, Muralidharan } 171184640e27SKaricheri, Muralidharan 171284640e27SKaricheri, Muralidharan for_each_module(netcp, intf_modpriv) { 171384640e27SKaricheri, Muralidharan module = intf_modpriv->netcp_module; 171484640e27SKaricheri, Muralidharan if (module->open) { 171584640e27SKaricheri, Muralidharan ret = module->open(intf_modpriv->module_priv, ndev); 171684640e27SKaricheri, Muralidharan if (ret != 0) { 171784640e27SKaricheri, Muralidharan dev_err(netcp->ndev_dev, "module open failed\n"); 171884640e27SKaricheri, Muralidharan goto fail_open; 171984640e27SKaricheri, Muralidharan } 172084640e27SKaricheri, Muralidharan } 172184640e27SKaricheri, Muralidharan } 172284640e27SKaricheri, Muralidharan 172384640e27SKaricheri, Muralidharan napi_enable(&netcp->rx_napi); 172484640e27SKaricheri, Muralidharan napi_enable(&netcp->tx_napi); 172584640e27SKaricheri, Muralidharan knav_queue_enable_notify(netcp->tx_compl_q); 172684640e27SKaricheri, Muralidharan knav_queue_enable_notify(netcp->rx_queue); 1727194ac06eSKaricheri, Muralidharan netcp_rxpool_refill(netcp); 172884640e27SKaricheri, Muralidharan netif_tx_wake_all_queues(ndev); 172984640e27SKaricheri, Muralidharan dev_dbg(netcp->ndev_dev, "netcp device %s opened\n", ndev->name); 173084640e27SKaricheri, Muralidharan return 0; 173184640e27SKaricheri, Muralidharan 173284640e27SKaricheri, Muralidharan fail_open: 173384640e27SKaricheri, Muralidharan for_each_module(netcp, intf_modpriv) { 173484640e27SKaricheri, Muralidharan module = intf_modpriv->netcp_module; 173584640e27SKaricheri, Muralidharan if (module->close) 173684640e27SKaricheri, Muralidharan module->close(intf_modpriv->module_priv, ndev); 173784640e27SKaricheri, Muralidharan } 173884640e27SKaricheri, Muralidharan 173984640e27SKaricheri, Muralidharan fail: 174084640e27SKaricheri, Muralidharan netcp_free_navigator_resources(netcp); 174184640e27SKaricheri, Muralidharan return ret; 174284640e27SKaricheri, Muralidharan } 174384640e27SKaricheri, Muralidharan 174484640e27SKaricheri, Muralidharan /* Close the device */ 174584640e27SKaricheri, Muralidharan static int netcp_ndo_stop(struct net_device *ndev) 174684640e27SKaricheri, Muralidharan { 174784640e27SKaricheri, Muralidharan struct netcp_intf *netcp = netdev_priv(ndev); 174884640e27SKaricheri, Muralidharan struct netcp_intf_modpriv *intf_modpriv; 174984640e27SKaricheri, Muralidharan struct netcp_module *module; 175084640e27SKaricheri, Muralidharan int err = 0; 175184640e27SKaricheri, Muralidharan 175284640e27SKaricheri, Muralidharan netif_tx_stop_all_queues(ndev); 175384640e27SKaricheri, Muralidharan netif_carrier_off(ndev); 175484640e27SKaricheri, Muralidharan netcp_addr_clear_mark(netcp); 175584640e27SKaricheri, Muralidharan netcp_addr_sweep_del(netcp); 175684640e27SKaricheri, Muralidharan knav_queue_disable_notify(netcp->rx_queue); 175784640e27SKaricheri, Muralidharan knav_queue_disable_notify(netcp->tx_compl_q); 175884640e27SKaricheri, Muralidharan napi_disable(&netcp->rx_napi); 175984640e27SKaricheri, Muralidharan napi_disable(&netcp->tx_napi); 176084640e27SKaricheri, Muralidharan 176184640e27SKaricheri, Muralidharan for_each_module(netcp, intf_modpriv) { 176284640e27SKaricheri, Muralidharan module = intf_modpriv->netcp_module; 176384640e27SKaricheri, Muralidharan if (module->close) { 176484640e27SKaricheri, Muralidharan err = module->close(intf_modpriv->module_priv, ndev); 176584640e27SKaricheri, Muralidharan if (err != 0) 176684640e27SKaricheri, Muralidharan dev_err(netcp->ndev_dev, "Close failed\n"); 176784640e27SKaricheri, Muralidharan } 176884640e27SKaricheri, Muralidharan } 176984640e27SKaricheri, Muralidharan 177084640e27SKaricheri, Muralidharan /* Recycle Rx descriptors from completion queue */ 177184640e27SKaricheri, Muralidharan netcp_empty_rx_queue(netcp); 177284640e27SKaricheri, Muralidharan 177384640e27SKaricheri, Muralidharan /* Recycle Tx descriptors from completion queue */ 177484640e27SKaricheri, Muralidharan netcp_process_tx_compl_packets(netcp, netcp->tx_pool_size); 177584640e27SKaricheri, Muralidharan 177684640e27SKaricheri, Muralidharan if (knav_pool_count(netcp->tx_pool) != netcp->tx_pool_size) 177784640e27SKaricheri, Muralidharan dev_err(netcp->ndev_dev, "Lost (%d) Tx descs\n", 177884640e27SKaricheri, Muralidharan netcp->tx_pool_size - knav_pool_count(netcp->tx_pool)); 177984640e27SKaricheri, Muralidharan 178084640e27SKaricheri, Muralidharan netcp_free_navigator_resources(netcp); 178184640e27SKaricheri, Muralidharan dev_dbg(netcp->ndev_dev, "netcp device %s stopped\n", ndev->name); 178284640e27SKaricheri, Muralidharan return 0; 178384640e27SKaricheri, Muralidharan } 178484640e27SKaricheri, Muralidharan 178584640e27SKaricheri, Muralidharan static int netcp_ndo_ioctl(struct net_device *ndev, 178684640e27SKaricheri, Muralidharan struct ifreq *req, int cmd) 178784640e27SKaricheri, Muralidharan { 178884640e27SKaricheri, Muralidharan struct netcp_intf *netcp = netdev_priv(ndev); 178984640e27SKaricheri, Muralidharan struct netcp_intf_modpriv *intf_modpriv; 179084640e27SKaricheri, Muralidharan struct netcp_module *module; 179184640e27SKaricheri, Muralidharan int ret = -1, err = -EOPNOTSUPP; 179284640e27SKaricheri, Muralidharan 179384640e27SKaricheri, Muralidharan if (!netif_running(ndev)) 179484640e27SKaricheri, Muralidharan return -EINVAL; 179584640e27SKaricheri, Muralidharan 179684640e27SKaricheri, Muralidharan for_each_module(netcp, intf_modpriv) { 179784640e27SKaricheri, Muralidharan module = intf_modpriv->netcp_module; 179884640e27SKaricheri, Muralidharan if (!module->ioctl) 179984640e27SKaricheri, Muralidharan continue; 180084640e27SKaricheri, Muralidharan 180184640e27SKaricheri, Muralidharan err = module->ioctl(intf_modpriv->module_priv, req, cmd); 180284640e27SKaricheri, Muralidharan if ((err < 0) && (err != -EOPNOTSUPP)) { 180384640e27SKaricheri, Muralidharan ret = err; 180484640e27SKaricheri, Muralidharan goto out; 180584640e27SKaricheri, Muralidharan } 180684640e27SKaricheri, Muralidharan if (err == 0) 180784640e27SKaricheri, Muralidharan ret = err; 180884640e27SKaricheri, Muralidharan } 180984640e27SKaricheri, Muralidharan 181084640e27SKaricheri, Muralidharan out: 181184640e27SKaricheri, Muralidharan return (ret == 0) ? 0 : err; 181284640e27SKaricheri, Muralidharan } 181384640e27SKaricheri, Muralidharan 181484640e27SKaricheri, Muralidharan static void netcp_ndo_tx_timeout(struct net_device *ndev) 181584640e27SKaricheri, Muralidharan { 181684640e27SKaricheri, Muralidharan struct netcp_intf *netcp = netdev_priv(ndev); 181784640e27SKaricheri, Muralidharan unsigned int descs = knav_pool_count(netcp->tx_pool); 181884640e27SKaricheri, Muralidharan 181984640e27SKaricheri, Muralidharan dev_err(netcp->ndev_dev, "transmit timed out tx descs(%d)\n", descs); 182084640e27SKaricheri, Muralidharan netcp_process_tx_compl_packets(netcp, netcp->tx_pool_size); 1821860e9538SFlorian Westphal netif_trans_update(ndev); 182284640e27SKaricheri, Muralidharan netif_tx_wake_all_queues(ndev); 182384640e27SKaricheri, Muralidharan } 182484640e27SKaricheri, Muralidharan 182584640e27SKaricheri, Muralidharan static int netcp_rx_add_vid(struct net_device *ndev, __be16 proto, u16 vid) 182684640e27SKaricheri, Muralidharan { 182784640e27SKaricheri, Muralidharan struct netcp_intf *netcp = netdev_priv(ndev); 182884640e27SKaricheri, Muralidharan struct netcp_intf_modpriv *intf_modpriv; 182984640e27SKaricheri, Muralidharan struct netcp_module *module; 18308ceaf361SKaricheri, Muralidharan unsigned long flags; 183184640e27SKaricheri, Muralidharan int err = 0; 183284640e27SKaricheri, Muralidharan 183384640e27SKaricheri, Muralidharan dev_dbg(netcp->ndev_dev, "adding rx vlan id: %d\n", vid); 183484640e27SKaricheri, Muralidharan 18358ceaf361SKaricheri, Muralidharan spin_lock_irqsave(&netcp->lock, flags); 183684640e27SKaricheri, Muralidharan for_each_module(netcp, intf_modpriv) { 183784640e27SKaricheri, Muralidharan module = intf_modpriv->netcp_module; 183884640e27SKaricheri, Muralidharan if ((module->add_vid) && (vid != 0)) { 183984640e27SKaricheri, Muralidharan err = module->add_vid(intf_modpriv->module_priv, vid); 184084640e27SKaricheri, Muralidharan if (err != 0) { 184184640e27SKaricheri, Muralidharan dev_err(netcp->ndev_dev, "Could not add vlan id = %d\n", 184284640e27SKaricheri, Muralidharan vid); 184384640e27SKaricheri, Muralidharan break; 184484640e27SKaricheri, Muralidharan } 184584640e27SKaricheri, Muralidharan } 184684640e27SKaricheri, Muralidharan } 18478ceaf361SKaricheri, Muralidharan spin_unlock_irqrestore(&netcp->lock, flags); 18488ceaf361SKaricheri, Muralidharan 184984640e27SKaricheri, Muralidharan return err; 185084640e27SKaricheri, Muralidharan } 185184640e27SKaricheri, Muralidharan 185284640e27SKaricheri, Muralidharan static int netcp_rx_kill_vid(struct net_device *ndev, __be16 proto, u16 vid) 185384640e27SKaricheri, Muralidharan { 185484640e27SKaricheri, Muralidharan struct netcp_intf *netcp = netdev_priv(ndev); 185584640e27SKaricheri, Muralidharan struct netcp_intf_modpriv *intf_modpriv; 185684640e27SKaricheri, Muralidharan struct netcp_module *module; 18578ceaf361SKaricheri, Muralidharan unsigned long flags; 185884640e27SKaricheri, Muralidharan int err = 0; 185984640e27SKaricheri, Muralidharan 186084640e27SKaricheri, Muralidharan dev_dbg(netcp->ndev_dev, "removing rx vlan id: %d\n", vid); 186184640e27SKaricheri, Muralidharan 18628ceaf361SKaricheri, Muralidharan spin_lock_irqsave(&netcp->lock, flags); 186384640e27SKaricheri, Muralidharan for_each_module(netcp, intf_modpriv) { 186484640e27SKaricheri, Muralidharan module = intf_modpriv->netcp_module; 186584640e27SKaricheri, Muralidharan if (module->del_vid) { 186684640e27SKaricheri, Muralidharan err = module->del_vid(intf_modpriv->module_priv, vid); 186784640e27SKaricheri, Muralidharan if (err != 0) { 186884640e27SKaricheri, Muralidharan dev_err(netcp->ndev_dev, "Could not delete vlan id = %d\n", 186984640e27SKaricheri, Muralidharan vid); 187084640e27SKaricheri, Muralidharan break; 187184640e27SKaricheri, Muralidharan } 187284640e27SKaricheri, Muralidharan } 187384640e27SKaricheri, Muralidharan } 18748ceaf361SKaricheri, Muralidharan spin_unlock_irqrestore(&netcp->lock, flags); 187584640e27SKaricheri, Muralidharan return err; 187684640e27SKaricheri, Muralidharan } 187784640e27SKaricheri, Muralidharan 18782572ac53SJiri Pirko static int netcp_setup_tc(struct net_device *dev, enum tc_setup_type type, 1879de4784caSJiri Pirko void *type_data) 188084640e27SKaricheri, Muralidharan { 1881de4784caSJiri Pirko struct tc_mqprio_qopt *mqprio = type_data; 188256f36acdSAmritha Nambiar u8 num_tc; 188384640e27SKaricheri, Muralidharan int i; 188484640e27SKaricheri, Muralidharan 188584640e27SKaricheri, Muralidharan /* setup tc must be called under rtnl lock */ 188684640e27SKaricheri, Muralidharan ASSERT_RTNL(); 188784640e27SKaricheri, Muralidharan 1888575ed7d3SNogah Frankel if (type != TC_SETUP_QDISC_MQPRIO) 188938cf0426SJiri Pirko return -EOPNOTSUPP; 1890e4c6734eSJohn Fastabend 1891de4784caSJiri Pirko mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS; 1892de4784caSJiri Pirko num_tc = mqprio->num_tc; 189356f36acdSAmritha Nambiar 189484640e27SKaricheri, Muralidharan /* Sanity-check the number of traffic classes requested */ 189584640e27SKaricheri, Muralidharan if ((dev->real_num_tx_queues <= 1) || 189656f36acdSAmritha Nambiar (dev->real_num_tx_queues < num_tc)) 189784640e27SKaricheri, Muralidharan return -EINVAL; 189884640e27SKaricheri, Muralidharan 189984640e27SKaricheri, Muralidharan /* Configure traffic class to queue mappings */ 190056f36acdSAmritha Nambiar if (num_tc) { 190156f36acdSAmritha Nambiar netdev_set_num_tc(dev, num_tc); 190256f36acdSAmritha Nambiar for (i = 0; i < num_tc; i++) 190384640e27SKaricheri, Muralidharan netdev_set_tc_queue(dev, i, 1, i); 190484640e27SKaricheri, Muralidharan } else { 190584640e27SKaricheri, Muralidharan netdev_reset_tc(dev); 190684640e27SKaricheri, Muralidharan } 190784640e27SKaricheri, Muralidharan 190884640e27SKaricheri, Muralidharan return 0; 190984640e27SKaricheri, Muralidharan } 191084640e27SKaricheri, Muralidharan 1911380043b9SKeerthy static void 19126a8162e9SMichael Scherban netcp_get_stats(struct net_device *ndev, struct rtnl_link_stats64 *stats) 19136a8162e9SMichael Scherban { 19146a8162e9SMichael Scherban struct netcp_intf *netcp = netdev_priv(ndev); 19156a8162e9SMichael Scherban struct netcp_stats *p = &netcp->stats; 19166a8162e9SMichael Scherban u64 rxpackets, rxbytes, txpackets, txbytes; 19176a8162e9SMichael Scherban unsigned int start; 19186a8162e9SMichael Scherban 19196a8162e9SMichael Scherban do { 19206a8162e9SMichael Scherban start = u64_stats_fetch_begin_irq(&p->syncp_rx); 19216a8162e9SMichael Scherban rxpackets = p->rx_packets; 19226a8162e9SMichael Scherban rxbytes = p->rx_bytes; 19236a8162e9SMichael Scherban } while (u64_stats_fetch_retry_irq(&p->syncp_rx, start)); 19246a8162e9SMichael Scherban 19256a8162e9SMichael Scherban do { 19266a8162e9SMichael Scherban start = u64_stats_fetch_begin_irq(&p->syncp_tx); 19276a8162e9SMichael Scherban txpackets = p->tx_packets; 19286a8162e9SMichael Scherban txbytes = p->tx_bytes; 19296a8162e9SMichael Scherban } while (u64_stats_fetch_retry_irq(&p->syncp_tx, start)); 19306a8162e9SMichael Scherban 19316a8162e9SMichael Scherban stats->rx_packets = rxpackets; 19326a8162e9SMichael Scherban stats->rx_bytes = rxbytes; 19336a8162e9SMichael Scherban stats->tx_packets = txpackets; 19346a8162e9SMichael Scherban stats->tx_bytes = txbytes; 19356a8162e9SMichael Scherban 19366a8162e9SMichael Scherban /* The following are stored as 32 bit */ 19376a8162e9SMichael Scherban stats->rx_errors = p->rx_errors; 19386a8162e9SMichael Scherban stats->rx_dropped = p->rx_dropped; 19396a8162e9SMichael Scherban stats->tx_dropped = p->tx_dropped; 19406a8162e9SMichael Scherban } 19416a8162e9SMichael Scherban 194284640e27SKaricheri, Muralidharan static const struct net_device_ops netcp_netdev_ops = { 194384640e27SKaricheri, Muralidharan .ndo_open = netcp_ndo_open, 194484640e27SKaricheri, Muralidharan .ndo_stop = netcp_ndo_stop, 194584640e27SKaricheri, Muralidharan .ndo_start_xmit = netcp_ndo_start_xmit, 194684640e27SKaricheri, Muralidharan .ndo_set_rx_mode = netcp_set_rx_mode, 194784640e27SKaricheri, Muralidharan .ndo_do_ioctl = netcp_ndo_ioctl, 19486a8162e9SMichael Scherban .ndo_get_stats64 = netcp_get_stats, 194984640e27SKaricheri, Muralidharan .ndo_set_mac_address = eth_mac_addr, 195084640e27SKaricheri, Muralidharan .ndo_validate_addr = eth_validate_addr, 195184640e27SKaricheri, Muralidharan .ndo_vlan_rx_add_vid = netcp_rx_add_vid, 195284640e27SKaricheri, Muralidharan .ndo_vlan_rx_kill_vid = netcp_rx_kill_vid, 195384640e27SKaricheri, Muralidharan .ndo_tx_timeout = netcp_ndo_tx_timeout, 1954a4ea8a3dSAlexander Duyck .ndo_select_queue = dev_pick_tx_zero, 195584640e27SKaricheri, Muralidharan .ndo_setup_tc = netcp_setup_tc, 195684640e27SKaricheri, Muralidharan }; 195784640e27SKaricheri, Muralidharan 195884640e27SKaricheri, Muralidharan static int netcp_create_interface(struct netcp_device *netcp_device, 195984640e27SKaricheri, Muralidharan struct device_node *node_interface) 196084640e27SKaricheri, Muralidharan { 196184640e27SKaricheri, Muralidharan struct device *dev = netcp_device->device; 196284640e27SKaricheri, Muralidharan struct device_node *node = dev->of_node; 196384640e27SKaricheri, Muralidharan struct netcp_intf *netcp; 196484640e27SKaricheri, Muralidharan struct net_device *ndev; 196584640e27SKaricheri, Muralidharan resource_size_t size; 196684640e27SKaricheri, Muralidharan struct resource res; 196784640e27SKaricheri, Muralidharan void __iomem *efuse = NULL; 196884640e27SKaricheri, Muralidharan u32 efuse_mac = 0; 196984640e27SKaricheri, Muralidharan const void *mac_addr; 197084640e27SKaricheri, Muralidharan u8 efuse_mac_addr[6]; 197184640e27SKaricheri, Muralidharan u32 temp[2]; 197284640e27SKaricheri, Muralidharan int ret = 0; 197384640e27SKaricheri, Muralidharan 197484640e27SKaricheri, Muralidharan ndev = alloc_etherdev_mqs(sizeof(*netcp), 1, 1); 197584640e27SKaricheri, Muralidharan if (!ndev) { 197684640e27SKaricheri, Muralidharan dev_err(dev, "Error allocating netdev\n"); 197784640e27SKaricheri, Muralidharan return -ENOMEM; 197884640e27SKaricheri, Muralidharan } 197984640e27SKaricheri, Muralidharan 198084640e27SKaricheri, Muralidharan ndev->features |= NETIF_F_SG; 198184640e27SKaricheri, Muralidharan ndev->features |= NETIF_F_HW_VLAN_CTAG_FILTER; 198284640e27SKaricheri, Muralidharan ndev->hw_features = ndev->features; 198384640e27SKaricheri, Muralidharan ndev->vlan_features |= NETIF_F_SG; 198484640e27SKaricheri, Muralidharan 198544770e11SJarod Wilson /* MTU range: 68 - 9486 */ 198644770e11SJarod Wilson ndev->min_mtu = ETH_MIN_MTU; 198744770e11SJarod Wilson ndev->max_mtu = NETCP_MAX_FRAME_SIZE - (ETH_HLEN + ETH_FCS_LEN); 198844770e11SJarod Wilson 198984640e27SKaricheri, Muralidharan netcp = netdev_priv(ndev); 199084640e27SKaricheri, Muralidharan spin_lock_init(&netcp->lock); 199184640e27SKaricheri, Muralidharan INIT_LIST_HEAD(&netcp->module_head); 199284640e27SKaricheri, Muralidharan INIT_LIST_HEAD(&netcp->txhook_list_head); 199384640e27SKaricheri, Muralidharan INIT_LIST_HEAD(&netcp->rxhook_list_head); 199484640e27SKaricheri, Muralidharan INIT_LIST_HEAD(&netcp->addr_list); 19956a8162e9SMichael Scherban u64_stats_init(&netcp->stats.syncp_rx); 19966a8162e9SMichael Scherban u64_stats_init(&netcp->stats.syncp_tx); 199784640e27SKaricheri, Muralidharan netcp->netcp_device = netcp_device; 199884640e27SKaricheri, Muralidharan netcp->dev = netcp_device->device; 199984640e27SKaricheri, Muralidharan netcp->ndev = ndev; 200084640e27SKaricheri, Muralidharan netcp->ndev_dev = &ndev->dev; 200184640e27SKaricheri, Muralidharan netcp->msg_enable = netif_msg_init(netcp_debug_level, NETCP_DEBUG); 200284640e27SKaricheri, Muralidharan netcp->tx_pause_threshold = MAX_SKB_FRAGS; 200384640e27SKaricheri, Muralidharan netcp->tx_resume_threshold = netcp->tx_pause_threshold; 200484640e27SKaricheri, Muralidharan netcp->node_interface = node_interface; 200584640e27SKaricheri, Muralidharan 200684640e27SKaricheri, Muralidharan ret = of_property_read_u32(node_interface, "efuse-mac", &efuse_mac); 200784640e27SKaricheri, Muralidharan if (efuse_mac) { 200884640e27SKaricheri, Muralidharan if (of_address_to_resource(node, NETCP_EFUSE_REG_INDEX, &res)) { 200984640e27SKaricheri, Muralidharan dev_err(dev, "could not find efuse-mac reg resource\n"); 201084640e27SKaricheri, Muralidharan ret = -ENODEV; 201184640e27SKaricheri, Muralidharan goto quit; 201284640e27SKaricheri, Muralidharan } 201384640e27SKaricheri, Muralidharan size = resource_size(&res); 201484640e27SKaricheri, Muralidharan 201584640e27SKaricheri, Muralidharan if (!devm_request_mem_region(dev, res.start, size, 201684640e27SKaricheri, Muralidharan dev_name(dev))) { 201784640e27SKaricheri, Muralidharan dev_err(dev, "could not reserve resource\n"); 201884640e27SKaricheri, Muralidharan ret = -ENOMEM; 201984640e27SKaricheri, Muralidharan goto quit; 202084640e27SKaricheri, Muralidharan } 202184640e27SKaricheri, Muralidharan 202284640e27SKaricheri, Muralidharan efuse = devm_ioremap_nocache(dev, res.start, size); 202384640e27SKaricheri, Muralidharan if (!efuse) { 202484640e27SKaricheri, Muralidharan dev_err(dev, "could not map resource\n"); 202584640e27SKaricheri, Muralidharan devm_release_mem_region(dev, res.start, size); 202684640e27SKaricheri, Muralidharan ret = -ENOMEM; 202784640e27SKaricheri, Muralidharan goto quit; 202884640e27SKaricheri, Muralidharan } 202984640e27SKaricheri, Muralidharan 203071382bc0SWingMan Kwok emac_arch_get_mac_addr(efuse_mac_addr, efuse, efuse_mac); 203184640e27SKaricheri, Muralidharan if (is_valid_ether_addr(efuse_mac_addr)) 203284640e27SKaricheri, Muralidharan ether_addr_copy(ndev->dev_addr, efuse_mac_addr); 203384640e27SKaricheri, Muralidharan else 20346c1f0a1fSJoe Perches eth_random_addr(ndev->dev_addr); 203584640e27SKaricheri, Muralidharan 203684640e27SKaricheri, Muralidharan devm_iounmap(dev, efuse); 203784640e27SKaricheri, Muralidharan devm_release_mem_region(dev, res.start, size); 203884640e27SKaricheri, Muralidharan } else { 203984640e27SKaricheri, Muralidharan mac_addr = of_get_mac_address(node_interface); 2040a51645f7SPetr Štetiar if (!IS_ERR(mac_addr)) 204184640e27SKaricheri, Muralidharan ether_addr_copy(ndev->dev_addr, mac_addr); 204284640e27SKaricheri, Muralidharan else 20436c1f0a1fSJoe Perches eth_random_addr(ndev->dev_addr); 204484640e27SKaricheri, Muralidharan } 204584640e27SKaricheri, Muralidharan 204684640e27SKaricheri, Muralidharan ret = of_property_read_string(node_interface, "rx-channel", 204784640e27SKaricheri, Muralidharan &netcp->dma_chan_name); 204884640e27SKaricheri, Muralidharan if (ret < 0) { 204984640e27SKaricheri, Muralidharan dev_err(dev, "missing \"rx-channel\" parameter\n"); 205084640e27SKaricheri, Muralidharan ret = -ENODEV; 205184640e27SKaricheri, Muralidharan goto quit; 205284640e27SKaricheri, Muralidharan } 205384640e27SKaricheri, Muralidharan 205484640e27SKaricheri, Muralidharan ret = of_property_read_u32(node_interface, "rx-queue", 205584640e27SKaricheri, Muralidharan &netcp->rx_queue_id); 205684640e27SKaricheri, Muralidharan if (ret < 0) { 205784640e27SKaricheri, Muralidharan dev_warn(dev, "missing \"rx-queue\" parameter\n"); 205884640e27SKaricheri, Muralidharan netcp->rx_queue_id = KNAV_QUEUE_QPEND; 205984640e27SKaricheri, Muralidharan } 206084640e27SKaricheri, Muralidharan 206184640e27SKaricheri, Muralidharan ret = of_property_read_u32_array(node_interface, "rx-queue-depth", 206284640e27SKaricheri, Muralidharan netcp->rx_queue_depths, 206384640e27SKaricheri, Muralidharan KNAV_DMA_FDQ_PER_CHAN); 206484640e27SKaricheri, Muralidharan if (ret < 0) { 206584640e27SKaricheri, Muralidharan dev_err(dev, "missing \"rx-queue-depth\" parameter\n"); 206684640e27SKaricheri, Muralidharan netcp->rx_queue_depths[0] = 128; 206784640e27SKaricheri, Muralidharan } 206884640e27SKaricheri, Muralidharan 206984640e27SKaricheri, Muralidharan ret = of_property_read_u32_array(node_interface, "rx-pool", temp, 2); 207084640e27SKaricheri, Muralidharan if (ret < 0) { 207184640e27SKaricheri, Muralidharan dev_err(dev, "missing \"rx-pool\" parameter\n"); 207284640e27SKaricheri, Muralidharan ret = -ENODEV; 207384640e27SKaricheri, Muralidharan goto quit; 207484640e27SKaricheri, Muralidharan } 207584640e27SKaricheri, Muralidharan netcp->rx_pool_size = temp[0]; 207684640e27SKaricheri, Muralidharan netcp->rx_pool_region_id = temp[1]; 207784640e27SKaricheri, Muralidharan 207884640e27SKaricheri, Muralidharan ret = of_property_read_u32_array(node_interface, "tx-pool", temp, 2); 207984640e27SKaricheri, Muralidharan if (ret < 0) { 208084640e27SKaricheri, Muralidharan dev_err(dev, "missing \"tx-pool\" parameter\n"); 208184640e27SKaricheri, Muralidharan ret = -ENODEV; 208284640e27SKaricheri, Muralidharan goto quit; 208384640e27SKaricheri, Muralidharan } 208484640e27SKaricheri, Muralidharan netcp->tx_pool_size = temp[0]; 208584640e27SKaricheri, Muralidharan netcp->tx_pool_region_id = temp[1]; 208684640e27SKaricheri, Muralidharan 208784640e27SKaricheri, Muralidharan if (netcp->tx_pool_size < MAX_SKB_FRAGS) { 208884640e27SKaricheri, Muralidharan dev_err(dev, "tx-pool size too small, must be atleast(%ld)\n", 208984640e27SKaricheri, Muralidharan MAX_SKB_FRAGS); 209084640e27SKaricheri, Muralidharan ret = -ENODEV; 209184640e27SKaricheri, Muralidharan goto quit; 209284640e27SKaricheri, Muralidharan } 209384640e27SKaricheri, Muralidharan 209484640e27SKaricheri, Muralidharan ret = of_property_read_u32(node_interface, "tx-completion-queue", 209584640e27SKaricheri, Muralidharan &netcp->tx_compl_qid); 209684640e27SKaricheri, Muralidharan if (ret < 0) { 209784640e27SKaricheri, Muralidharan dev_warn(dev, "missing \"tx-completion-queue\" parameter\n"); 209884640e27SKaricheri, Muralidharan netcp->tx_compl_qid = KNAV_QUEUE_QPEND; 209984640e27SKaricheri, Muralidharan } 210084640e27SKaricheri, Muralidharan 210184640e27SKaricheri, Muralidharan /* NAPI register */ 210284640e27SKaricheri, Muralidharan netif_napi_add(ndev, &netcp->rx_napi, netcp_rx_poll, NETCP_NAPI_WEIGHT); 2103d64b5e85SEric Dumazet netif_tx_napi_add(ndev, &netcp->tx_napi, netcp_tx_poll, NETCP_NAPI_WEIGHT); 210484640e27SKaricheri, Muralidharan 210584640e27SKaricheri, Muralidharan /* Register the network device */ 210684640e27SKaricheri, Muralidharan ndev->dev_id = 0; 210784640e27SKaricheri, Muralidharan ndev->watchdog_timeo = NETCP_TX_TIMEOUT; 210884640e27SKaricheri, Muralidharan ndev->netdev_ops = &netcp_netdev_ops; 210984640e27SKaricheri, Muralidharan SET_NETDEV_DEV(ndev, dev); 211084640e27SKaricheri, Muralidharan 211184640e27SKaricheri, Muralidharan list_add_tail(&netcp->interface_list, &netcp_device->interface_head); 211284640e27SKaricheri, Muralidharan return 0; 211384640e27SKaricheri, Muralidharan 211484640e27SKaricheri, Muralidharan quit: 211584640e27SKaricheri, Muralidharan free_netdev(ndev); 211684640e27SKaricheri, Muralidharan return ret; 211784640e27SKaricheri, Muralidharan } 211884640e27SKaricheri, Muralidharan 211984640e27SKaricheri, Muralidharan static void netcp_delete_interface(struct netcp_device *netcp_device, 212084640e27SKaricheri, Muralidharan struct net_device *ndev) 212184640e27SKaricheri, Muralidharan { 212284640e27SKaricheri, Muralidharan struct netcp_intf_modpriv *intf_modpriv, *tmp; 212384640e27SKaricheri, Muralidharan struct netcp_intf *netcp = netdev_priv(ndev); 212484640e27SKaricheri, Muralidharan struct netcp_module *module; 212584640e27SKaricheri, Muralidharan 212684640e27SKaricheri, Muralidharan dev_dbg(netcp_device->device, "Removing interface \"%s\"\n", 212784640e27SKaricheri, Muralidharan ndev->name); 212884640e27SKaricheri, Muralidharan 212984640e27SKaricheri, Muralidharan /* Notify each of the modules that the interface is going away */ 213084640e27SKaricheri, Muralidharan list_for_each_entry_safe(intf_modpriv, tmp, &netcp->module_head, 213184640e27SKaricheri, Muralidharan intf_list) { 213284640e27SKaricheri, Muralidharan module = intf_modpriv->netcp_module; 213384640e27SKaricheri, Muralidharan dev_dbg(netcp_device->device, "Releasing module \"%s\"\n", 213484640e27SKaricheri, Muralidharan module->name); 213584640e27SKaricheri, Muralidharan if (module->release) 213684640e27SKaricheri, Muralidharan module->release(intf_modpriv->module_priv); 213784640e27SKaricheri, Muralidharan list_del(&intf_modpriv->intf_list); 213884640e27SKaricheri, Muralidharan } 213984640e27SKaricheri, Muralidharan WARN(!list_empty(&netcp->module_head), "%s interface module list is not empty!\n", 214084640e27SKaricheri, Muralidharan ndev->name); 214184640e27SKaricheri, Muralidharan 214284640e27SKaricheri, Muralidharan list_del(&netcp->interface_list); 214384640e27SKaricheri, Muralidharan 214484640e27SKaricheri, Muralidharan of_node_put(netcp->node_interface); 214584640e27SKaricheri, Muralidharan unregister_netdev(ndev); 214684640e27SKaricheri, Muralidharan free_netdev(ndev); 214784640e27SKaricheri, Muralidharan } 214884640e27SKaricheri, Muralidharan 214984640e27SKaricheri, Muralidharan static int netcp_probe(struct platform_device *pdev) 215084640e27SKaricheri, Muralidharan { 215184640e27SKaricheri, Muralidharan struct device_node *node = pdev->dev.of_node; 215284640e27SKaricheri, Muralidharan struct netcp_intf *netcp_intf, *netcp_tmp; 215384640e27SKaricheri, Muralidharan struct device_node *child, *interfaces; 215484640e27SKaricheri, Muralidharan struct netcp_device *netcp_device; 215584640e27SKaricheri, Muralidharan struct device *dev = &pdev->dev; 2156c52b6782SMurali Karicheri struct netcp_module *module; 215784640e27SKaricheri, Muralidharan int ret; 215884640e27SKaricheri, Muralidharan 215921f706bbSMurali Karicheri if (!knav_dma_device_ready() || 216021f706bbSMurali Karicheri !knav_qmss_device_ready()) 216121f706bbSMurali Karicheri return -EPROBE_DEFER; 216221f706bbSMurali Karicheri 216384640e27SKaricheri, Muralidharan if (!node) { 216484640e27SKaricheri, Muralidharan dev_err(dev, "could not find device info\n"); 216584640e27SKaricheri, Muralidharan return -ENODEV; 216684640e27SKaricheri, Muralidharan } 216784640e27SKaricheri, Muralidharan 216884640e27SKaricheri, Muralidharan /* Allocate a new NETCP device instance */ 216984640e27SKaricheri, Muralidharan netcp_device = devm_kzalloc(dev, sizeof(*netcp_device), GFP_KERNEL); 217084640e27SKaricheri, Muralidharan if (!netcp_device) 217184640e27SKaricheri, Muralidharan return -ENOMEM; 217284640e27SKaricheri, Muralidharan 217384640e27SKaricheri, Muralidharan pm_runtime_enable(&pdev->dev); 217484640e27SKaricheri, Muralidharan ret = pm_runtime_get_sync(&pdev->dev); 217584640e27SKaricheri, Muralidharan if (ret < 0) { 217684640e27SKaricheri, Muralidharan dev_err(dev, "Failed to enable NETCP power-domain\n"); 217784640e27SKaricheri, Muralidharan pm_runtime_disable(&pdev->dev); 217884640e27SKaricheri, Muralidharan return ret; 217984640e27SKaricheri, Muralidharan } 218084640e27SKaricheri, Muralidharan 218184640e27SKaricheri, Muralidharan /* Initialize the NETCP device instance */ 218284640e27SKaricheri, Muralidharan INIT_LIST_HEAD(&netcp_device->interface_head); 218384640e27SKaricheri, Muralidharan INIT_LIST_HEAD(&netcp_device->modpriv_head); 218484640e27SKaricheri, Muralidharan netcp_device->device = dev; 218584640e27SKaricheri, Muralidharan platform_set_drvdata(pdev, netcp_device); 218684640e27SKaricheri, Muralidharan 218784640e27SKaricheri, Muralidharan /* create interfaces */ 218884640e27SKaricheri, Muralidharan interfaces = of_get_child_by_name(node, "netcp-interfaces"); 218984640e27SKaricheri, Muralidharan if (!interfaces) { 219084640e27SKaricheri, Muralidharan dev_err(dev, "could not find netcp-interfaces node\n"); 219184640e27SKaricheri, Muralidharan ret = -ENODEV; 219284640e27SKaricheri, Muralidharan goto probe_quit; 219384640e27SKaricheri, Muralidharan } 219484640e27SKaricheri, Muralidharan 219584640e27SKaricheri, Muralidharan for_each_available_child_of_node(interfaces, child) { 219684640e27SKaricheri, Muralidharan ret = netcp_create_interface(netcp_device, child); 219784640e27SKaricheri, Muralidharan if (ret) { 219821c328dcSRob Herring dev_err(dev, "could not create interface(%pOFn)\n", 219921c328dcSRob Herring child); 220084640e27SKaricheri, Muralidharan goto probe_quit_interface; 220184640e27SKaricheri, Muralidharan } 220284640e27SKaricheri, Muralidharan } 220384640e27SKaricheri, Muralidharan 2204e2897b82SWei Yongjun of_node_put(interfaces); 2205e2897b82SWei Yongjun 220684640e27SKaricheri, Muralidharan /* Add the device instance to the list */ 220784640e27SKaricheri, Muralidharan list_add_tail(&netcp_device->device_list, &netcp_devices); 220884640e27SKaricheri, Muralidharan 2209c52b6782SMurali Karicheri /* Probe & attach any modules already registered */ 2210c52b6782SMurali Karicheri mutex_lock(&netcp_modules_lock); 2211c52b6782SMurali Karicheri for_each_netcp_module(module) { 2212c52b6782SMurali Karicheri ret = netcp_module_probe(netcp_device, module); 2213c52b6782SMurali Karicheri if (ret < 0) 2214c52b6782SMurali Karicheri dev_err(dev, "module(%s) probe failed\n", module->name); 2215c52b6782SMurali Karicheri } 2216c52b6782SMurali Karicheri mutex_unlock(&netcp_modules_lock); 221784640e27SKaricheri, Muralidharan return 0; 221884640e27SKaricheri, Muralidharan 221984640e27SKaricheri, Muralidharan probe_quit_interface: 222084640e27SKaricheri, Muralidharan list_for_each_entry_safe(netcp_intf, netcp_tmp, 222184640e27SKaricheri, Muralidharan &netcp_device->interface_head, 222284640e27SKaricheri, Muralidharan interface_list) { 222384640e27SKaricheri, Muralidharan netcp_delete_interface(netcp_device, netcp_intf->ndev); 222484640e27SKaricheri, Muralidharan } 222584640e27SKaricheri, Muralidharan 2226e2897b82SWei Yongjun of_node_put(interfaces); 2227e2897b82SWei Yongjun 222884640e27SKaricheri, Muralidharan probe_quit: 222984640e27SKaricheri, Muralidharan pm_runtime_put_sync(&pdev->dev); 223084640e27SKaricheri, Muralidharan pm_runtime_disable(&pdev->dev); 223184640e27SKaricheri, Muralidharan platform_set_drvdata(pdev, NULL); 223284640e27SKaricheri, Muralidharan return ret; 223384640e27SKaricheri, Muralidharan } 223484640e27SKaricheri, Muralidharan 223584640e27SKaricheri, Muralidharan static int netcp_remove(struct platform_device *pdev) 223684640e27SKaricheri, Muralidharan { 223784640e27SKaricheri, Muralidharan struct netcp_device *netcp_device = platform_get_drvdata(pdev); 223801a03099SKaricheri, Muralidharan struct netcp_intf *netcp_intf, *netcp_tmp; 223984640e27SKaricheri, Muralidharan struct netcp_inst_modpriv *inst_modpriv, *tmp; 224084640e27SKaricheri, Muralidharan struct netcp_module *module; 224184640e27SKaricheri, Muralidharan 224284640e27SKaricheri, Muralidharan list_for_each_entry_safe(inst_modpriv, tmp, &netcp_device->modpriv_head, 224384640e27SKaricheri, Muralidharan inst_list) { 224484640e27SKaricheri, Muralidharan module = inst_modpriv->netcp_module; 224584640e27SKaricheri, Muralidharan dev_dbg(&pdev->dev, "Removing module \"%s\"\n", module->name); 224684640e27SKaricheri, Muralidharan module->remove(netcp_device, inst_modpriv->module_priv); 224784640e27SKaricheri, Muralidharan list_del(&inst_modpriv->inst_list); 224884640e27SKaricheri, Muralidharan } 224984640e27SKaricheri, Muralidharan 225001a03099SKaricheri, Muralidharan /* now that all modules are removed, clean up the interfaces */ 225101a03099SKaricheri, Muralidharan list_for_each_entry_safe(netcp_intf, netcp_tmp, 225201a03099SKaricheri, Muralidharan &netcp_device->interface_head, 225301a03099SKaricheri, Muralidharan interface_list) { 225401a03099SKaricheri, Muralidharan netcp_delete_interface(netcp_device, netcp_intf->ndev); 225501a03099SKaricheri, Muralidharan } 225601a03099SKaricheri, Muralidharan 225701a03099SKaricheri, Muralidharan WARN(!list_empty(&netcp_device->interface_head), 225801a03099SKaricheri, Muralidharan "%s interface list not empty!\n", pdev->name); 225984640e27SKaricheri, Muralidharan 226084640e27SKaricheri, Muralidharan pm_runtime_put_sync(&pdev->dev); 226184640e27SKaricheri, Muralidharan pm_runtime_disable(&pdev->dev); 226284640e27SKaricheri, Muralidharan platform_set_drvdata(pdev, NULL); 226384640e27SKaricheri, Muralidharan return 0; 226484640e27SKaricheri, Muralidharan } 226584640e27SKaricheri, Muralidharan 22661156c965SFabian Frederick static const struct of_device_id of_match[] = { 226784640e27SKaricheri, Muralidharan { .compatible = "ti,netcp-1.0", }, 226884640e27SKaricheri, Muralidharan {}, 226984640e27SKaricheri, Muralidharan }; 227084640e27SKaricheri, Muralidharan MODULE_DEVICE_TABLE(of, of_match); 227184640e27SKaricheri, Muralidharan 227284640e27SKaricheri, Muralidharan static struct platform_driver netcp_driver = { 227384640e27SKaricheri, Muralidharan .driver = { 227484640e27SKaricheri, Muralidharan .name = "netcp-1.0", 227584640e27SKaricheri, Muralidharan .of_match_table = of_match, 227684640e27SKaricheri, Muralidharan }, 227784640e27SKaricheri, Muralidharan .probe = netcp_probe, 227884640e27SKaricheri, Muralidharan .remove = netcp_remove, 227984640e27SKaricheri, Muralidharan }; 228084640e27SKaricheri, Muralidharan module_platform_driver(netcp_driver); 228184640e27SKaricheri, Muralidharan 228284640e27SKaricheri, Muralidharan MODULE_LICENSE("GPL v2"); 228384640e27SKaricheri, Muralidharan MODULE_DESCRIPTION("TI NETCP driver for Keystone SOCs"); 228484640e27SKaricheri, Muralidharan MODULE_AUTHOR("Sandeep Nair <sandeep_n@ti.com"); 2285