184640e27SKaricheri, Muralidharan /* 284640e27SKaricheri, Muralidharan * Keystone NetCP Core driver 384640e27SKaricheri, Muralidharan * 484640e27SKaricheri, Muralidharan * Copyright (C) 2014 Texas Instruments Incorporated 584640e27SKaricheri, Muralidharan * Authors: Sandeep Nair <sandeep_n@ti.com> 684640e27SKaricheri, Muralidharan * Sandeep Paulraj <s-paulraj@ti.com> 784640e27SKaricheri, Muralidharan * Cyril Chemparathy <cyril@ti.com> 884640e27SKaricheri, Muralidharan * Santosh Shilimkar <santosh.shilimkar@ti.com> 984640e27SKaricheri, Muralidharan * Murali Karicheri <m-karicheri2@ti.com> 1084640e27SKaricheri, Muralidharan * Wingman Kwok <w-kwok2@ti.com> 1184640e27SKaricheri, Muralidharan * 1284640e27SKaricheri, Muralidharan * This program is free software; you can redistribute it and/or 1384640e27SKaricheri, Muralidharan * modify it under the terms of the GNU General Public License as 1484640e27SKaricheri, Muralidharan * published by the Free Software Foundation version 2. 1584640e27SKaricheri, Muralidharan * 1684640e27SKaricheri, Muralidharan * This program is distributed "as is" WITHOUT ANY WARRANTY of any 1784640e27SKaricheri, Muralidharan * kind, whether express or implied; without even the implied warranty 1884640e27SKaricheri, Muralidharan * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 1984640e27SKaricheri, Muralidharan * GNU General Public License for more details. 2084640e27SKaricheri, Muralidharan */ 2184640e27SKaricheri, Muralidharan 2284640e27SKaricheri, Muralidharan #include <linux/io.h> 2384640e27SKaricheri, Muralidharan #include <linux/module.h> 2484640e27SKaricheri, Muralidharan #include <linux/of_net.h> 2584640e27SKaricheri, Muralidharan #include <linux/of_address.h> 2684640e27SKaricheri, Muralidharan #include <linux/if_vlan.h> 2784640e27SKaricheri, Muralidharan #include <linux/pm_runtime.h> 2884640e27SKaricheri, Muralidharan #include <linux/platform_device.h> 2984640e27SKaricheri, Muralidharan #include <linux/soc/ti/knav_qmss.h> 3084640e27SKaricheri, Muralidharan #include <linux/soc/ti/knav_dma.h> 3184640e27SKaricheri, Muralidharan 3284640e27SKaricheri, Muralidharan #include "netcp.h" 3384640e27SKaricheri, Muralidharan 3484640e27SKaricheri, Muralidharan #define NETCP_SOP_OFFSET (NET_IP_ALIGN + NET_SKB_PAD) 3584640e27SKaricheri, Muralidharan #define NETCP_NAPI_WEIGHT 64 3684640e27SKaricheri, Muralidharan #define NETCP_TX_TIMEOUT (5 * HZ) 37866b8b18SWingMan Kwok #define NETCP_PACKET_SIZE (ETH_FRAME_LEN + ETH_FCS_LEN) 3884640e27SKaricheri, Muralidharan #define NETCP_MIN_PACKET_SIZE ETH_ZLEN 3984640e27SKaricheri, Muralidharan #define NETCP_MAX_MCAST_ADDR 16 4084640e27SKaricheri, Muralidharan 4184640e27SKaricheri, Muralidharan #define NETCP_EFUSE_REG_INDEX 0 4284640e27SKaricheri, Muralidharan 4384640e27SKaricheri, Muralidharan #define NETCP_MOD_PROBE_SKIPPED 1 4484640e27SKaricheri, Muralidharan #define NETCP_MOD_PROBE_FAILED 2 4584640e27SKaricheri, Muralidharan 4684640e27SKaricheri, Muralidharan #define NETCP_DEBUG (NETIF_MSG_HW | NETIF_MSG_WOL | \ 4784640e27SKaricheri, Muralidharan NETIF_MSG_DRV | NETIF_MSG_LINK | \ 4884640e27SKaricheri, Muralidharan NETIF_MSG_IFUP | NETIF_MSG_INTR | \ 4984640e27SKaricheri, Muralidharan NETIF_MSG_PROBE | NETIF_MSG_TIMER | \ 5084640e27SKaricheri, Muralidharan NETIF_MSG_IFDOWN | NETIF_MSG_RX_ERR | \ 5184640e27SKaricheri, Muralidharan NETIF_MSG_TX_ERR | NETIF_MSG_TX_DONE | \ 5284640e27SKaricheri, Muralidharan NETIF_MSG_PKTDATA | NETIF_MSG_TX_QUEUED | \ 5384640e27SKaricheri, Muralidharan NETIF_MSG_RX_STATUS) 5484640e27SKaricheri, Muralidharan 5571382bc0SWingMan Kwok #define NETCP_EFUSE_ADDR_SWAP 2 5671382bc0SWingMan Kwok 5784640e27SKaricheri, Muralidharan #define knav_queue_get_id(q) knav_queue_device_control(q, \ 5884640e27SKaricheri, Muralidharan KNAV_QUEUE_GET_ID, (unsigned long)NULL) 5984640e27SKaricheri, Muralidharan 6084640e27SKaricheri, Muralidharan #define knav_queue_enable_notify(q) knav_queue_device_control(q, \ 6184640e27SKaricheri, Muralidharan KNAV_QUEUE_ENABLE_NOTIFY, \ 6284640e27SKaricheri, Muralidharan (unsigned long)NULL) 6384640e27SKaricheri, Muralidharan 6484640e27SKaricheri, Muralidharan #define knav_queue_disable_notify(q) knav_queue_device_control(q, \ 6584640e27SKaricheri, Muralidharan KNAV_QUEUE_DISABLE_NOTIFY, \ 6684640e27SKaricheri, Muralidharan (unsigned long)NULL) 6784640e27SKaricheri, Muralidharan 6884640e27SKaricheri, Muralidharan #define knav_queue_get_count(q) knav_queue_device_control(q, \ 6984640e27SKaricheri, Muralidharan KNAV_QUEUE_GET_COUNT, (unsigned long)NULL) 7084640e27SKaricheri, Muralidharan 7184640e27SKaricheri, Muralidharan #define for_each_netcp_module(module) \ 7284640e27SKaricheri, Muralidharan list_for_each_entry(module, &netcp_modules, module_list) 7384640e27SKaricheri, Muralidharan 7484640e27SKaricheri, Muralidharan #define for_each_netcp_device_module(netcp_device, inst_modpriv) \ 7584640e27SKaricheri, Muralidharan list_for_each_entry(inst_modpriv, \ 7684640e27SKaricheri, Muralidharan &((netcp_device)->modpriv_head), inst_list) 7784640e27SKaricheri, Muralidharan 7884640e27SKaricheri, Muralidharan #define for_each_module(netcp, intf_modpriv) \ 7984640e27SKaricheri, Muralidharan list_for_each_entry(intf_modpriv, &netcp->module_head, intf_list) 8084640e27SKaricheri, Muralidharan 8184640e27SKaricheri, Muralidharan /* Module management structures */ 8284640e27SKaricheri, Muralidharan struct netcp_device { 8384640e27SKaricheri, Muralidharan struct list_head device_list; 8484640e27SKaricheri, Muralidharan struct list_head interface_head; 8584640e27SKaricheri, Muralidharan struct list_head modpriv_head; 8684640e27SKaricheri, Muralidharan struct device *device; 8784640e27SKaricheri, Muralidharan }; 8884640e27SKaricheri, Muralidharan 8984640e27SKaricheri, Muralidharan struct netcp_inst_modpriv { 9084640e27SKaricheri, Muralidharan struct netcp_device *netcp_device; 9184640e27SKaricheri, Muralidharan struct netcp_module *netcp_module; 9284640e27SKaricheri, Muralidharan struct list_head inst_list; 9384640e27SKaricheri, Muralidharan void *module_priv; 9484640e27SKaricheri, Muralidharan }; 9584640e27SKaricheri, Muralidharan 9684640e27SKaricheri, Muralidharan struct netcp_intf_modpriv { 9784640e27SKaricheri, Muralidharan struct netcp_intf *netcp_priv; 9884640e27SKaricheri, Muralidharan struct netcp_module *netcp_module; 9984640e27SKaricheri, Muralidharan struct list_head intf_list; 10084640e27SKaricheri, Muralidharan void *module_priv; 10184640e27SKaricheri, Muralidharan }; 10284640e27SKaricheri, Muralidharan 1036246168bSWingMan Kwok struct netcp_tx_cb { 1046246168bSWingMan Kwok void *ts_context; 1056246168bSWingMan Kwok void (*txtstamp)(void *context, struct sk_buff *skb); 1066246168bSWingMan Kwok }; 1076246168bSWingMan Kwok 10884640e27SKaricheri, Muralidharan static LIST_HEAD(netcp_devices); 10984640e27SKaricheri, Muralidharan static LIST_HEAD(netcp_modules); 11084640e27SKaricheri, Muralidharan static DEFINE_MUTEX(netcp_modules_lock); 11184640e27SKaricheri, Muralidharan 11284640e27SKaricheri, Muralidharan static int netcp_debug_level = -1; 11384640e27SKaricheri, Muralidharan module_param(netcp_debug_level, int, 0); 11484640e27SKaricheri, Muralidharan MODULE_PARM_DESC(netcp_debug_level, "Netcp debug level (NETIF_MSG bits) (0=none,...,16=all)"); 11584640e27SKaricheri, Muralidharan 11684640e27SKaricheri, Muralidharan /* Helper functions - Get/Set */ 11789907779SArnd Bergmann static void get_pkt_info(dma_addr_t *buff, u32 *buff_len, dma_addr_t *ndesc, 11884640e27SKaricheri, Muralidharan struct knav_dma_desc *desc) 11984640e27SKaricheri, Muralidharan { 12089907779SArnd Bergmann *buff_len = le32_to_cpu(desc->buff_len); 12189907779SArnd Bergmann *buff = le32_to_cpu(desc->buff); 12289907779SArnd Bergmann *ndesc = le32_to_cpu(desc->next_desc); 12384640e27SKaricheri, Muralidharan } 12484640e27SKaricheri, Muralidharan 12569d707d0SKaricheri, Muralidharan static void get_desc_info(u32 *desc_info, u32 *pkt_info, 12669d707d0SKaricheri, Muralidharan struct knav_dma_desc *desc) 12769d707d0SKaricheri, Muralidharan { 12869d707d0SKaricheri, Muralidharan *desc_info = le32_to_cpu(desc->desc_info); 12969d707d0SKaricheri, Muralidharan *pkt_info = le32_to_cpu(desc->packet_info); 13069d707d0SKaricheri, Muralidharan } 13169d707d0SKaricheri, Muralidharan 13206324481SKaricheri, Muralidharan static u32 get_sw_data(int index, struct knav_dma_desc *desc) 13384640e27SKaricheri, Muralidharan { 134b1cb86aeSKaricheri, Muralidharan /* No Endian conversion needed as this data is untouched by hw */ 13506324481SKaricheri, Muralidharan return desc->sw_data[index]; 13684640e27SKaricheri, Muralidharan } 13784640e27SKaricheri, Muralidharan 13806324481SKaricheri, Muralidharan /* use these macros to get sw data */ 13906324481SKaricheri, Muralidharan #define GET_SW_DATA0(desc) get_sw_data(0, desc) 14006324481SKaricheri, Muralidharan #define GET_SW_DATA1(desc) get_sw_data(1, desc) 14106324481SKaricheri, Muralidharan #define GET_SW_DATA2(desc) get_sw_data(2, desc) 14206324481SKaricheri, Muralidharan #define GET_SW_DATA3(desc) get_sw_data(3, desc) 14389907779SArnd Bergmann 14489907779SArnd Bergmann static void get_org_pkt_info(dma_addr_t *buff, u32 *buff_len, 14584640e27SKaricheri, Muralidharan struct knav_dma_desc *desc) 14684640e27SKaricheri, Muralidharan { 14789907779SArnd Bergmann *buff = le32_to_cpu(desc->orig_buff); 14889907779SArnd Bergmann *buff_len = le32_to_cpu(desc->orig_len); 14984640e27SKaricheri, Muralidharan } 15084640e27SKaricheri, Muralidharan 15189907779SArnd Bergmann static void get_words(dma_addr_t *words, int num_words, __le32 *desc) 15284640e27SKaricheri, Muralidharan { 15384640e27SKaricheri, Muralidharan int i; 15484640e27SKaricheri, Muralidharan 15584640e27SKaricheri, Muralidharan for (i = 0; i < num_words; i++) 15689907779SArnd Bergmann words[i] = le32_to_cpu(desc[i]); 15784640e27SKaricheri, Muralidharan } 15884640e27SKaricheri, Muralidharan 15989907779SArnd Bergmann static void set_pkt_info(dma_addr_t buff, u32 buff_len, u32 ndesc, 16084640e27SKaricheri, Muralidharan struct knav_dma_desc *desc) 16184640e27SKaricheri, Muralidharan { 16289907779SArnd Bergmann desc->buff_len = cpu_to_le32(buff_len); 16389907779SArnd Bergmann desc->buff = cpu_to_le32(buff); 16489907779SArnd Bergmann desc->next_desc = cpu_to_le32(ndesc); 16584640e27SKaricheri, Muralidharan } 16684640e27SKaricheri, Muralidharan 16784640e27SKaricheri, Muralidharan static void set_desc_info(u32 desc_info, u32 pkt_info, 16884640e27SKaricheri, Muralidharan struct knav_dma_desc *desc) 16984640e27SKaricheri, Muralidharan { 17089907779SArnd Bergmann desc->desc_info = cpu_to_le32(desc_info); 17189907779SArnd Bergmann desc->packet_info = cpu_to_le32(pkt_info); 17284640e27SKaricheri, Muralidharan } 17384640e27SKaricheri, Muralidharan 17406324481SKaricheri, Muralidharan static void set_sw_data(int index, u32 data, struct knav_dma_desc *desc) 17584640e27SKaricheri, Muralidharan { 176b1cb86aeSKaricheri, Muralidharan /* No Endian conversion needed as this data is untouched by hw */ 17706324481SKaricheri, Muralidharan desc->sw_data[index] = data; 17884640e27SKaricheri, Muralidharan } 17984640e27SKaricheri, Muralidharan 18006324481SKaricheri, Muralidharan /* use these macros to set sw data */ 18106324481SKaricheri, Muralidharan #define SET_SW_DATA0(data, desc) set_sw_data(0, data, desc) 18206324481SKaricheri, Muralidharan #define SET_SW_DATA1(data, desc) set_sw_data(1, data, desc) 18306324481SKaricheri, Muralidharan #define SET_SW_DATA2(data, desc) set_sw_data(2, data, desc) 18406324481SKaricheri, Muralidharan #define SET_SW_DATA3(data, desc) set_sw_data(3, data, desc) 18506324481SKaricheri, Muralidharan 18689907779SArnd Bergmann static void set_org_pkt_info(dma_addr_t buff, u32 buff_len, 18784640e27SKaricheri, Muralidharan struct knav_dma_desc *desc) 18884640e27SKaricheri, Muralidharan { 18989907779SArnd Bergmann desc->orig_buff = cpu_to_le32(buff); 19089907779SArnd Bergmann desc->orig_len = cpu_to_le32(buff_len); 19184640e27SKaricheri, Muralidharan } 19284640e27SKaricheri, Muralidharan 19389907779SArnd Bergmann static void set_words(u32 *words, int num_words, __le32 *desc) 19484640e27SKaricheri, Muralidharan { 19584640e27SKaricheri, Muralidharan int i; 19684640e27SKaricheri, Muralidharan 19784640e27SKaricheri, Muralidharan for (i = 0; i < num_words; i++) 19889907779SArnd Bergmann desc[i] = cpu_to_le32(words[i]); 19984640e27SKaricheri, Muralidharan } 20084640e27SKaricheri, Muralidharan 20184640e27SKaricheri, Muralidharan /* Read the e-fuse value as 32 bit values to be endian independent */ 20271382bc0SWingMan Kwok static int emac_arch_get_mac_addr(char *x, void __iomem *efuse_mac, u32 swap) 20384640e27SKaricheri, Muralidharan { 20484640e27SKaricheri, Muralidharan unsigned int addr0, addr1; 20584640e27SKaricheri, Muralidharan 20684640e27SKaricheri, Muralidharan addr1 = readl(efuse_mac + 4); 20784640e27SKaricheri, Muralidharan addr0 = readl(efuse_mac); 20884640e27SKaricheri, Muralidharan 20971382bc0SWingMan Kwok switch (swap) { 21071382bc0SWingMan Kwok case NETCP_EFUSE_ADDR_SWAP: 21171382bc0SWingMan Kwok addr0 = addr1; 21271382bc0SWingMan Kwok addr1 = readl(efuse_mac); 21371382bc0SWingMan Kwok break; 21471382bc0SWingMan Kwok default: 21571382bc0SWingMan Kwok break; 21671382bc0SWingMan Kwok } 21771382bc0SWingMan Kwok 21884640e27SKaricheri, Muralidharan x[0] = (addr1 & 0x0000ff00) >> 8; 21984640e27SKaricheri, Muralidharan x[1] = addr1 & 0x000000ff; 22084640e27SKaricheri, Muralidharan x[2] = (addr0 & 0xff000000) >> 24; 22184640e27SKaricheri, Muralidharan x[3] = (addr0 & 0x00ff0000) >> 16; 22284640e27SKaricheri, Muralidharan x[4] = (addr0 & 0x0000ff00) >> 8; 22384640e27SKaricheri, Muralidharan x[5] = addr0 & 0x000000ff; 22484640e27SKaricheri, Muralidharan 22584640e27SKaricheri, Muralidharan return 0; 22684640e27SKaricheri, Muralidharan } 22784640e27SKaricheri, Muralidharan 22884640e27SKaricheri, Muralidharan /* Module management routines */ 22984640e27SKaricheri, Muralidharan static int netcp_register_interface(struct netcp_intf *netcp) 23084640e27SKaricheri, Muralidharan { 23184640e27SKaricheri, Muralidharan int ret; 23284640e27SKaricheri, Muralidharan 23384640e27SKaricheri, Muralidharan ret = register_netdev(netcp->ndev); 23484640e27SKaricheri, Muralidharan if (!ret) 23584640e27SKaricheri, Muralidharan netcp->netdev_registered = true; 23684640e27SKaricheri, Muralidharan return ret; 23784640e27SKaricheri, Muralidharan } 23884640e27SKaricheri, Muralidharan 23984640e27SKaricheri, Muralidharan static int netcp_module_probe(struct netcp_device *netcp_device, 24084640e27SKaricheri, Muralidharan struct netcp_module *module) 24184640e27SKaricheri, Muralidharan { 24284640e27SKaricheri, Muralidharan struct device *dev = netcp_device->device; 24384640e27SKaricheri, Muralidharan struct device_node *devices, *interface, *node = dev->of_node; 24484640e27SKaricheri, Muralidharan struct device_node *child; 24584640e27SKaricheri, Muralidharan struct netcp_inst_modpriv *inst_modpriv; 24684640e27SKaricheri, Muralidharan struct netcp_intf *netcp_intf; 24784640e27SKaricheri, Muralidharan struct netcp_module *tmp; 24884640e27SKaricheri, Muralidharan bool primary_module_registered = false; 24984640e27SKaricheri, Muralidharan int ret; 25084640e27SKaricheri, Muralidharan 25184640e27SKaricheri, Muralidharan /* Find this module in the sub-tree for this device */ 25284640e27SKaricheri, Muralidharan devices = of_get_child_by_name(node, "netcp-devices"); 25384640e27SKaricheri, Muralidharan if (!devices) { 25484640e27SKaricheri, Muralidharan dev_err(dev, "could not find netcp-devices node\n"); 25584640e27SKaricheri, Muralidharan return NETCP_MOD_PROBE_SKIPPED; 25684640e27SKaricheri, Muralidharan } 25784640e27SKaricheri, Muralidharan 25884640e27SKaricheri, Muralidharan for_each_available_child_of_node(devices, child) { 25921c328dcSRob Herring const char *name; 26021c328dcSRob Herring char node_name[32]; 26184640e27SKaricheri, Muralidharan 2621f43f400SMurali Karicheri if (of_property_read_string(child, "label", &name) < 0) { 26321c328dcSRob Herring snprintf(node_name, sizeof(node_name), "%pOFn", child); 26421c328dcSRob Herring name = node_name; 26521c328dcSRob Herring } 26684640e27SKaricheri, Muralidharan if (!strcasecmp(module->name, name)) 26784640e27SKaricheri, Muralidharan break; 26884640e27SKaricheri, Muralidharan } 26984640e27SKaricheri, Muralidharan 27084640e27SKaricheri, Muralidharan of_node_put(devices); 27184640e27SKaricheri, Muralidharan /* If module not used for this device, skip it */ 27284640e27SKaricheri, Muralidharan if (!child) { 27384640e27SKaricheri, Muralidharan dev_warn(dev, "module(%s) not used for device\n", module->name); 27484640e27SKaricheri, Muralidharan return NETCP_MOD_PROBE_SKIPPED; 27584640e27SKaricheri, Muralidharan } 27684640e27SKaricheri, Muralidharan 27784640e27SKaricheri, Muralidharan inst_modpriv = devm_kzalloc(dev, sizeof(*inst_modpriv), GFP_KERNEL); 27884640e27SKaricheri, Muralidharan if (!inst_modpriv) { 27984640e27SKaricheri, Muralidharan of_node_put(child); 28084640e27SKaricheri, Muralidharan return -ENOMEM; 28184640e27SKaricheri, Muralidharan } 28284640e27SKaricheri, Muralidharan 28384640e27SKaricheri, Muralidharan inst_modpriv->netcp_device = netcp_device; 28484640e27SKaricheri, Muralidharan inst_modpriv->netcp_module = module; 28584640e27SKaricheri, Muralidharan list_add_tail(&inst_modpriv->inst_list, &netcp_device->modpriv_head); 28684640e27SKaricheri, Muralidharan 28784640e27SKaricheri, Muralidharan ret = module->probe(netcp_device, dev, child, 28884640e27SKaricheri, Muralidharan &inst_modpriv->module_priv); 28984640e27SKaricheri, Muralidharan of_node_put(child); 29084640e27SKaricheri, Muralidharan if (ret) { 29184640e27SKaricheri, Muralidharan dev_err(dev, "Probe of module(%s) failed with %d\n", 29284640e27SKaricheri, Muralidharan module->name, ret); 29384640e27SKaricheri, Muralidharan list_del(&inst_modpriv->inst_list); 29484640e27SKaricheri, Muralidharan devm_kfree(dev, inst_modpriv); 29584640e27SKaricheri, Muralidharan return NETCP_MOD_PROBE_FAILED; 29684640e27SKaricheri, Muralidharan } 29784640e27SKaricheri, Muralidharan 29884640e27SKaricheri, Muralidharan /* Attach modules only if the primary module is probed */ 29984640e27SKaricheri, Muralidharan for_each_netcp_module(tmp) { 30084640e27SKaricheri, Muralidharan if (tmp->primary) 30184640e27SKaricheri, Muralidharan primary_module_registered = true; 30284640e27SKaricheri, Muralidharan } 30384640e27SKaricheri, Muralidharan 30484640e27SKaricheri, Muralidharan if (!primary_module_registered) 30584640e27SKaricheri, Muralidharan return 0; 30684640e27SKaricheri, Muralidharan 30784640e27SKaricheri, Muralidharan /* Attach module to interfaces */ 30884640e27SKaricheri, Muralidharan list_for_each_entry(netcp_intf, &netcp_device->interface_head, 30984640e27SKaricheri, Muralidharan interface_list) { 31084640e27SKaricheri, Muralidharan struct netcp_intf_modpriv *intf_modpriv; 31184640e27SKaricheri, Muralidharan 31284640e27SKaricheri, Muralidharan intf_modpriv = devm_kzalloc(dev, sizeof(*intf_modpriv), 31384640e27SKaricheri, Muralidharan GFP_KERNEL); 31484640e27SKaricheri, Muralidharan if (!intf_modpriv) 31584640e27SKaricheri, Muralidharan return -ENOMEM; 31684640e27SKaricheri, Muralidharan 31784640e27SKaricheri, Muralidharan interface = of_parse_phandle(netcp_intf->node_interface, 31884640e27SKaricheri, Muralidharan module->name, 0); 31984640e27SKaricheri, Muralidharan 320915c5857SKaricheri, Muralidharan if (!interface) { 321915c5857SKaricheri, Muralidharan devm_kfree(dev, intf_modpriv); 322915c5857SKaricheri, Muralidharan continue; 323915c5857SKaricheri, Muralidharan } 324915c5857SKaricheri, Muralidharan 32584640e27SKaricheri, Muralidharan intf_modpriv->netcp_priv = netcp_intf; 32684640e27SKaricheri, Muralidharan intf_modpriv->netcp_module = module; 32784640e27SKaricheri, Muralidharan list_add_tail(&intf_modpriv->intf_list, 32884640e27SKaricheri, Muralidharan &netcp_intf->module_head); 32984640e27SKaricheri, Muralidharan 33084640e27SKaricheri, Muralidharan ret = module->attach(inst_modpriv->module_priv, 33184640e27SKaricheri, Muralidharan netcp_intf->ndev, interface, 33284640e27SKaricheri, Muralidharan &intf_modpriv->module_priv); 33384640e27SKaricheri, Muralidharan of_node_put(interface); 33484640e27SKaricheri, Muralidharan if (ret) { 33584640e27SKaricheri, Muralidharan dev_dbg(dev, "Attach of module %s declined with %d\n", 33684640e27SKaricheri, Muralidharan module->name, ret); 33784640e27SKaricheri, Muralidharan list_del(&intf_modpriv->intf_list); 33884640e27SKaricheri, Muralidharan devm_kfree(dev, intf_modpriv); 33984640e27SKaricheri, Muralidharan continue; 34084640e27SKaricheri, Muralidharan } 34184640e27SKaricheri, Muralidharan } 342736532a0SKaricheri, Muralidharan 343736532a0SKaricheri, Muralidharan /* Now register the interface with netdev */ 344736532a0SKaricheri, Muralidharan list_for_each_entry(netcp_intf, 345736532a0SKaricheri, Muralidharan &netcp_device->interface_head, 346736532a0SKaricheri, Muralidharan interface_list) { 347736532a0SKaricheri, Muralidharan /* If interface not registered then register now */ 348736532a0SKaricheri, Muralidharan if (!netcp_intf->netdev_registered) { 349736532a0SKaricheri, Muralidharan ret = netcp_register_interface(netcp_intf); 350736532a0SKaricheri, Muralidharan if (ret) 351736532a0SKaricheri, Muralidharan return -ENODEV; 352736532a0SKaricheri, Muralidharan } 353736532a0SKaricheri, Muralidharan } 35484640e27SKaricheri, Muralidharan return 0; 35584640e27SKaricheri, Muralidharan } 35684640e27SKaricheri, Muralidharan 35784640e27SKaricheri, Muralidharan int netcp_register_module(struct netcp_module *module) 35884640e27SKaricheri, Muralidharan { 35984640e27SKaricheri, Muralidharan struct netcp_device *netcp_device; 36084640e27SKaricheri, Muralidharan struct netcp_module *tmp; 36184640e27SKaricheri, Muralidharan int ret; 36284640e27SKaricheri, Muralidharan 36384640e27SKaricheri, Muralidharan if (!module->name) { 36484640e27SKaricheri, Muralidharan WARN(1, "error registering netcp module: no name\n"); 36584640e27SKaricheri, Muralidharan return -EINVAL; 36684640e27SKaricheri, Muralidharan } 36784640e27SKaricheri, Muralidharan 36884640e27SKaricheri, Muralidharan if (!module->probe) { 36984640e27SKaricheri, Muralidharan WARN(1, "error registering netcp module: no probe\n"); 37084640e27SKaricheri, Muralidharan return -EINVAL; 37184640e27SKaricheri, Muralidharan } 37284640e27SKaricheri, Muralidharan 37384640e27SKaricheri, Muralidharan mutex_lock(&netcp_modules_lock); 37484640e27SKaricheri, Muralidharan 37584640e27SKaricheri, Muralidharan for_each_netcp_module(tmp) { 37684640e27SKaricheri, Muralidharan if (!strcasecmp(tmp->name, module->name)) { 37784640e27SKaricheri, Muralidharan mutex_unlock(&netcp_modules_lock); 37884640e27SKaricheri, Muralidharan return -EEXIST; 37984640e27SKaricheri, Muralidharan } 38084640e27SKaricheri, Muralidharan } 38184640e27SKaricheri, Muralidharan list_add_tail(&module->module_list, &netcp_modules); 38284640e27SKaricheri, Muralidharan 38384640e27SKaricheri, Muralidharan list_for_each_entry(netcp_device, &netcp_devices, device_list) { 38484640e27SKaricheri, Muralidharan ret = netcp_module_probe(netcp_device, module); 38584640e27SKaricheri, Muralidharan if (ret < 0) 38684640e27SKaricheri, Muralidharan goto fail; 38784640e27SKaricheri, Muralidharan } 38884640e27SKaricheri, Muralidharan mutex_unlock(&netcp_modules_lock); 38984640e27SKaricheri, Muralidharan return 0; 39084640e27SKaricheri, Muralidharan 39184640e27SKaricheri, Muralidharan fail: 39284640e27SKaricheri, Muralidharan mutex_unlock(&netcp_modules_lock); 39384640e27SKaricheri, Muralidharan netcp_unregister_module(module); 39484640e27SKaricheri, Muralidharan return ret; 39584640e27SKaricheri, Muralidharan } 39658c11b5fSKaricheri, Muralidharan EXPORT_SYMBOL_GPL(netcp_register_module); 39784640e27SKaricheri, Muralidharan 39884640e27SKaricheri, Muralidharan static void netcp_release_module(struct netcp_device *netcp_device, 39984640e27SKaricheri, Muralidharan struct netcp_module *module) 40084640e27SKaricheri, Muralidharan { 40184640e27SKaricheri, Muralidharan struct netcp_inst_modpriv *inst_modpriv, *inst_tmp; 40284640e27SKaricheri, Muralidharan struct netcp_intf *netcp_intf, *netcp_tmp; 40384640e27SKaricheri, Muralidharan struct device *dev = netcp_device->device; 40484640e27SKaricheri, Muralidharan 40584640e27SKaricheri, Muralidharan /* Release the module from each interface */ 40684640e27SKaricheri, Muralidharan list_for_each_entry_safe(netcp_intf, netcp_tmp, 40784640e27SKaricheri, Muralidharan &netcp_device->interface_head, 40884640e27SKaricheri, Muralidharan interface_list) { 40984640e27SKaricheri, Muralidharan struct netcp_intf_modpriv *intf_modpriv, *intf_tmp; 41084640e27SKaricheri, Muralidharan 41184640e27SKaricheri, Muralidharan list_for_each_entry_safe(intf_modpriv, intf_tmp, 41284640e27SKaricheri, Muralidharan &netcp_intf->module_head, 41384640e27SKaricheri, Muralidharan intf_list) { 41484640e27SKaricheri, Muralidharan if (intf_modpriv->netcp_module == module) { 41584640e27SKaricheri, Muralidharan module->release(intf_modpriv->module_priv); 41684640e27SKaricheri, Muralidharan list_del(&intf_modpriv->intf_list); 41784640e27SKaricheri, Muralidharan devm_kfree(dev, intf_modpriv); 41884640e27SKaricheri, Muralidharan break; 41984640e27SKaricheri, Muralidharan } 42084640e27SKaricheri, Muralidharan } 42184640e27SKaricheri, Muralidharan } 42284640e27SKaricheri, Muralidharan 42384640e27SKaricheri, Muralidharan /* Remove the module from each instance */ 42484640e27SKaricheri, Muralidharan list_for_each_entry_safe(inst_modpriv, inst_tmp, 42584640e27SKaricheri, Muralidharan &netcp_device->modpriv_head, inst_list) { 42684640e27SKaricheri, Muralidharan if (inst_modpriv->netcp_module == module) { 42784640e27SKaricheri, Muralidharan module->remove(netcp_device, 42884640e27SKaricheri, Muralidharan inst_modpriv->module_priv); 42984640e27SKaricheri, Muralidharan list_del(&inst_modpriv->inst_list); 43084640e27SKaricheri, Muralidharan devm_kfree(dev, inst_modpriv); 43184640e27SKaricheri, Muralidharan break; 43284640e27SKaricheri, Muralidharan } 43384640e27SKaricheri, Muralidharan } 43484640e27SKaricheri, Muralidharan } 43584640e27SKaricheri, Muralidharan 43684640e27SKaricheri, Muralidharan void netcp_unregister_module(struct netcp_module *module) 43784640e27SKaricheri, Muralidharan { 43884640e27SKaricheri, Muralidharan struct netcp_device *netcp_device; 43984640e27SKaricheri, Muralidharan struct netcp_module *module_tmp; 44084640e27SKaricheri, Muralidharan 44184640e27SKaricheri, Muralidharan mutex_lock(&netcp_modules_lock); 44284640e27SKaricheri, Muralidharan 44384640e27SKaricheri, Muralidharan list_for_each_entry(netcp_device, &netcp_devices, device_list) { 44484640e27SKaricheri, Muralidharan netcp_release_module(netcp_device, module); 44584640e27SKaricheri, Muralidharan } 44684640e27SKaricheri, Muralidharan 44784640e27SKaricheri, Muralidharan /* Remove the module from the module list */ 44884640e27SKaricheri, Muralidharan for_each_netcp_module(module_tmp) { 44984640e27SKaricheri, Muralidharan if (module == module_tmp) { 45084640e27SKaricheri, Muralidharan list_del(&module->module_list); 45184640e27SKaricheri, Muralidharan break; 45284640e27SKaricheri, Muralidharan } 45384640e27SKaricheri, Muralidharan } 45484640e27SKaricheri, Muralidharan 45584640e27SKaricheri, Muralidharan mutex_unlock(&netcp_modules_lock); 45684640e27SKaricheri, Muralidharan } 45758c11b5fSKaricheri, Muralidharan EXPORT_SYMBOL_GPL(netcp_unregister_module); 45884640e27SKaricheri, Muralidharan 45984640e27SKaricheri, Muralidharan void *netcp_module_get_intf_data(struct netcp_module *module, 46084640e27SKaricheri, Muralidharan struct netcp_intf *intf) 46184640e27SKaricheri, Muralidharan { 46284640e27SKaricheri, Muralidharan struct netcp_intf_modpriv *intf_modpriv; 46384640e27SKaricheri, Muralidharan 46484640e27SKaricheri, Muralidharan list_for_each_entry(intf_modpriv, &intf->module_head, intf_list) 46584640e27SKaricheri, Muralidharan if (intf_modpriv->netcp_module == module) 46684640e27SKaricheri, Muralidharan return intf_modpriv->module_priv; 46784640e27SKaricheri, Muralidharan return NULL; 46884640e27SKaricheri, Muralidharan } 46958c11b5fSKaricheri, Muralidharan EXPORT_SYMBOL_GPL(netcp_module_get_intf_data); 47084640e27SKaricheri, Muralidharan 47184640e27SKaricheri, Muralidharan /* Module TX and RX Hook management */ 47284640e27SKaricheri, Muralidharan struct netcp_hook_list { 47384640e27SKaricheri, Muralidharan struct list_head list; 47484640e27SKaricheri, Muralidharan netcp_hook_rtn *hook_rtn; 47584640e27SKaricheri, Muralidharan void *hook_data; 47684640e27SKaricheri, Muralidharan int order; 47784640e27SKaricheri, Muralidharan }; 47884640e27SKaricheri, Muralidharan 47984640e27SKaricheri, Muralidharan int netcp_register_txhook(struct netcp_intf *netcp_priv, int order, 48084640e27SKaricheri, Muralidharan netcp_hook_rtn *hook_rtn, void *hook_data) 48184640e27SKaricheri, Muralidharan { 48284640e27SKaricheri, Muralidharan struct netcp_hook_list *entry; 48384640e27SKaricheri, Muralidharan struct netcp_hook_list *next; 48484640e27SKaricheri, Muralidharan unsigned long flags; 48584640e27SKaricheri, Muralidharan 48684640e27SKaricheri, Muralidharan entry = devm_kzalloc(netcp_priv->dev, sizeof(*entry), GFP_KERNEL); 48784640e27SKaricheri, Muralidharan if (!entry) 48884640e27SKaricheri, Muralidharan return -ENOMEM; 48984640e27SKaricheri, Muralidharan 49084640e27SKaricheri, Muralidharan entry->hook_rtn = hook_rtn; 49184640e27SKaricheri, Muralidharan entry->hook_data = hook_data; 49284640e27SKaricheri, Muralidharan entry->order = order; 49384640e27SKaricheri, Muralidharan 49484640e27SKaricheri, Muralidharan spin_lock_irqsave(&netcp_priv->lock, flags); 49584640e27SKaricheri, Muralidharan list_for_each_entry(next, &netcp_priv->txhook_list_head, list) { 49684640e27SKaricheri, Muralidharan if (next->order > order) 49784640e27SKaricheri, Muralidharan break; 49884640e27SKaricheri, Muralidharan } 49984640e27SKaricheri, Muralidharan __list_add(&entry->list, next->list.prev, &next->list); 50084640e27SKaricheri, Muralidharan spin_unlock_irqrestore(&netcp_priv->lock, flags); 50184640e27SKaricheri, Muralidharan 50284640e27SKaricheri, Muralidharan return 0; 50384640e27SKaricheri, Muralidharan } 50458c11b5fSKaricheri, Muralidharan EXPORT_SYMBOL_GPL(netcp_register_txhook); 50584640e27SKaricheri, Muralidharan 50684640e27SKaricheri, Muralidharan int netcp_unregister_txhook(struct netcp_intf *netcp_priv, int order, 50784640e27SKaricheri, Muralidharan netcp_hook_rtn *hook_rtn, void *hook_data) 50884640e27SKaricheri, Muralidharan { 50984640e27SKaricheri, Muralidharan struct netcp_hook_list *next, *n; 51084640e27SKaricheri, Muralidharan unsigned long flags; 51184640e27SKaricheri, Muralidharan 51284640e27SKaricheri, Muralidharan spin_lock_irqsave(&netcp_priv->lock, flags); 51384640e27SKaricheri, Muralidharan list_for_each_entry_safe(next, n, &netcp_priv->txhook_list_head, list) { 51484640e27SKaricheri, Muralidharan if ((next->order == order) && 51584640e27SKaricheri, Muralidharan (next->hook_rtn == hook_rtn) && 51684640e27SKaricheri, Muralidharan (next->hook_data == hook_data)) { 51784640e27SKaricheri, Muralidharan list_del(&next->list); 51884640e27SKaricheri, Muralidharan spin_unlock_irqrestore(&netcp_priv->lock, flags); 51984640e27SKaricheri, Muralidharan devm_kfree(netcp_priv->dev, next); 52084640e27SKaricheri, Muralidharan return 0; 52184640e27SKaricheri, Muralidharan } 52284640e27SKaricheri, Muralidharan } 52384640e27SKaricheri, Muralidharan spin_unlock_irqrestore(&netcp_priv->lock, flags); 52484640e27SKaricheri, Muralidharan return -ENOENT; 52584640e27SKaricheri, Muralidharan } 52658c11b5fSKaricheri, Muralidharan EXPORT_SYMBOL_GPL(netcp_unregister_txhook); 52784640e27SKaricheri, Muralidharan 52884640e27SKaricheri, Muralidharan int netcp_register_rxhook(struct netcp_intf *netcp_priv, int order, 52984640e27SKaricheri, Muralidharan netcp_hook_rtn *hook_rtn, void *hook_data) 53084640e27SKaricheri, Muralidharan { 53184640e27SKaricheri, Muralidharan struct netcp_hook_list *entry; 53284640e27SKaricheri, Muralidharan struct netcp_hook_list *next; 53384640e27SKaricheri, Muralidharan unsigned long flags; 53484640e27SKaricheri, Muralidharan 53584640e27SKaricheri, Muralidharan entry = devm_kzalloc(netcp_priv->dev, sizeof(*entry), GFP_KERNEL); 53684640e27SKaricheri, Muralidharan if (!entry) 53784640e27SKaricheri, Muralidharan return -ENOMEM; 53884640e27SKaricheri, Muralidharan 53984640e27SKaricheri, Muralidharan entry->hook_rtn = hook_rtn; 54084640e27SKaricheri, Muralidharan entry->hook_data = hook_data; 54184640e27SKaricheri, Muralidharan entry->order = order; 54284640e27SKaricheri, Muralidharan 54384640e27SKaricheri, Muralidharan spin_lock_irqsave(&netcp_priv->lock, flags); 54484640e27SKaricheri, Muralidharan list_for_each_entry(next, &netcp_priv->rxhook_list_head, list) { 54584640e27SKaricheri, Muralidharan if (next->order > order) 54684640e27SKaricheri, Muralidharan break; 54784640e27SKaricheri, Muralidharan } 54884640e27SKaricheri, Muralidharan __list_add(&entry->list, next->list.prev, &next->list); 54984640e27SKaricheri, Muralidharan spin_unlock_irqrestore(&netcp_priv->lock, flags); 55084640e27SKaricheri, Muralidharan 55184640e27SKaricheri, Muralidharan return 0; 55284640e27SKaricheri, Muralidharan } 5536246168bSWingMan Kwok EXPORT_SYMBOL_GPL(netcp_register_rxhook); 55484640e27SKaricheri, Muralidharan 55584640e27SKaricheri, Muralidharan int netcp_unregister_rxhook(struct netcp_intf *netcp_priv, int order, 55684640e27SKaricheri, Muralidharan netcp_hook_rtn *hook_rtn, void *hook_data) 55784640e27SKaricheri, Muralidharan { 55884640e27SKaricheri, Muralidharan struct netcp_hook_list *next, *n; 55984640e27SKaricheri, Muralidharan unsigned long flags; 56084640e27SKaricheri, Muralidharan 56184640e27SKaricheri, Muralidharan spin_lock_irqsave(&netcp_priv->lock, flags); 56284640e27SKaricheri, Muralidharan list_for_each_entry_safe(next, n, &netcp_priv->rxhook_list_head, list) { 56384640e27SKaricheri, Muralidharan if ((next->order == order) && 56484640e27SKaricheri, Muralidharan (next->hook_rtn == hook_rtn) && 56584640e27SKaricheri, Muralidharan (next->hook_data == hook_data)) { 56684640e27SKaricheri, Muralidharan list_del(&next->list); 56784640e27SKaricheri, Muralidharan spin_unlock_irqrestore(&netcp_priv->lock, flags); 56884640e27SKaricheri, Muralidharan devm_kfree(netcp_priv->dev, next); 56984640e27SKaricheri, Muralidharan return 0; 57084640e27SKaricheri, Muralidharan } 57184640e27SKaricheri, Muralidharan } 57284640e27SKaricheri, Muralidharan spin_unlock_irqrestore(&netcp_priv->lock, flags); 57384640e27SKaricheri, Muralidharan 57484640e27SKaricheri, Muralidharan return -ENOENT; 57584640e27SKaricheri, Muralidharan } 5766246168bSWingMan Kwok EXPORT_SYMBOL_GPL(netcp_unregister_rxhook); 57784640e27SKaricheri, Muralidharan 57884640e27SKaricheri, Muralidharan static void netcp_frag_free(bool is_frag, void *ptr) 57984640e27SKaricheri, Muralidharan { 58084640e27SKaricheri, Muralidharan if (is_frag) 5817d525c4eSAlexander Duyck skb_free_frag(ptr); 58284640e27SKaricheri, Muralidharan else 58384640e27SKaricheri, Muralidharan kfree(ptr); 58484640e27SKaricheri, Muralidharan } 58584640e27SKaricheri, Muralidharan 58684640e27SKaricheri, Muralidharan static void netcp_free_rx_desc_chain(struct netcp_intf *netcp, 58784640e27SKaricheri, Muralidharan struct knav_dma_desc *desc) 58884640e27SKaricheri, Muralidharan { 58984640e27SKaricheri, Muralidharan struct knav_dma_desc *ndesc; 59084640e27SKaricheri, Muralidharan dma_addr_t dma_desc, dma_buf; 59184640e27SKaricheri, Muralidharan unsigned int buf_len, dma_sz = sizeof(*ndesc); 59284640e27SKaricheri, Muralidharan void *buf_ptr; 593958d104eSArnd Bergmann u32 tmp; 59484640e27SKaricheri, Muralidharan 59584640e27SKaricheri, Muralidharan get_words(&dma_desc, 1, &desc->next_desc); 59684640e27SKaricheri, Muralidharan 59784640e27SKaricheri, Muralidharan while (dma_desc) { 59884640e27SKaricheri, Muralidharan ndesc = knav_pool_desc_unmap(netcp->rx_pool, dma_desc, dma_sz); 59984640e27SKaricheri, Muralidharan if (unlikely(!ndesc)) { 60084640e27SKaricheri, Muralidharan dev_err(netcp->ndev_dev, "failed to unmap Rx desc\n"); 60184640e27SKaricheri, Muralidharan break; 60284640e27SKaricheri, Muralidharan } 603958d104eSArnd Bergmann get_pkt_info(&dma_buf, &tmp, &dma_desc, ndesc); 60406324481SKaricheri, Muralidharan /* warning!!!! We are retrieving the virtual ptr in the sw_data 60506324481SKaricheri, Muralidharan * field as a 32bit value. Will not work on 64bit machines 60606324481SKaricheri, Muralidharan */ 60706324481SKaricheri, Muralidharan buf_ptr = (void *)GET_SW_DATA0(ndesc); 60806324481SKaricheri, Muralidharan buf_len = (int)GET_SW_DATA1(desc); 60984640e27SKaricheri, Muralidharan dma_unmap_page(netcp->dev, dma_buf, PAGE_SIZE, DMA_FROM_DEVICE); 61084640e27SKaricheri, Muralidharan __free_page(buf_ptr); 61184640e27SKaricheri, Muralidharan knav_pool_desc_put(netcp->rx_pool, desc); 61284640e27SKaricheri, Muralidharan } 61306324481SKaricheri, Muralidharan /* warning!!!! We are retrieving the virtual ptr in the sw_data 61406324481SKaricheri, Muralidharan * field as a 32bit value. Will not work on 64bit machines 61506324481SKaricheri, Muralidharan */ 61606324481SKaricheri, Muralidharan buf_ptr = (void *)GET_SW_DATA0(desc); 61706324481SKaricheri, Muralidharan buf_len = (int)GET_SW_DATA1(desc); 61889907779SArnd Bergmann 61984640e27SKaricheri, Muralidharan if (buf_ptr) 62084640e27SKaricheri, Muralidharan netcp_frag_free(buf_len <= PAGE_SIZE, buf_ptr); 62184640e27SKaricheri, Muralidharan knav_pool_desc_put(netcp->rx_pool, desc); 62284640e27SKaricheri, Muralidharan } 62384640e27SKaricheri, Muralidharan 62484640e27SKaricheri, Muralidharan static void netcp_empty_rx_queue(struct netcp_intf *netcp) 62584640e27SKaricheri, Muralidharan { 6266a8162e9SMichael Scherban struct netcp_stats *rx_stats = &netcp->stats; 62784640e27SKaricheri, Muralidharan struct knav_dma_desc *desc; 62884640e27SKaricheri, Muralidharan unsigned int dma_sz; 62984640e27SKaricheri, Muralidharan dma_addr_t dma; 63084640e27SKaricheri, Muralidharan 63184640e27SKaricheri, Muralidharan for (; ;) { 63284640e27SKaricheri, Muralidharan dma = knav_queue_pop(netcp->rx_queue, &dma_sz); 63384640e27SKaricheri, Muralidharan if (!dma) 63484640e27SKaricheri, Muralidharan break; 63584640e27SKaricheri, Muralidharan 63684640e27SKaricheri, Muralidharan desc = knav_pool_desc_unmap(netcp->rx_pool, dma, dma_sz); 63784640e27SKaricheri, Muralidharan if (unlikely(!desc)) { 63884640e27SKaricheri, Muralidharan dev_err(netcp->ndev_dev, "%s: failed to unmap Rx desc\n", 63984640e27SKaricheri, Muralidharan __func__); 6406a8162e9SMichael Scherban rx_stats->rx_errors++; 64184640e27SKaricheri, Muralidharan continue; 64284640e27SKaricheri, Muralidharan } 64384640e27SKaricheri, Muralidharan netcp_free_rx_desc_chain(netcp, desc); 6446a8162e9SMichael Scherban rx_stats->rx_dropped++; 64584640e27SKaricheri, Muralidharan } 64684640e27SKaricheri, Muralidharan } 64784640e27SKaricheri, Muralidharan 64884640e27SKaricheri, Muralidharan static int netcp_process_one_rx_packet(struct netcp_intf *netcp) 64984640e27SKaricheri, Muralidharan { 6506a8162e9SMichael Scherban struct netcp_stats *rx_stats = &netcp->stats; 65184640e27SKaricheri, Muralidharan unsigned int dma_sz, buf_len, org_buf_len; 65284640e27SKaricheri, Muralidharan struct knav_dma_desc *desc, *ndesc; 65384640e27SKaricheri, Muralidharan unsigned int pkt_sz = 0, accum_sz; 65484640e27SKaricheri, Muralidharan struct netcp_hook_list *rx_hook; 65584640e27SKaricheri, Muralidharan dma_addr_t dma_desc, dma_buff; 65684640e27SKaricheri, Muralidharan struct netcp_packet p_info; 65784640e27SKaricheri, Muralidharan struct sk_buff *skb; 65884640e27SKaricheri, Muralidharan void *org_buf_ptr; 65969d707d0SKaricheri, Muralidharan u32 tmp; 66084640e27SKaricheri, Muralidharan 66184640e27SKaricheri, Muralidharan dma_desc = knav_queue_pop(netcp->rx_queue, &dma_sz); 66284640e27SKaricheri, Muralidharan if (!dma_desc) 66384640e27SKaricheri, Muralidharan return -1; 66484640e27SKaricheri, Muralidharan 66584640e27SKaricheri, Muralidharan desc = knav_pool_desc_unmap(netcp->rx_pool, dma_desc, dma_sz); 66684640e27SKaricheri, Muralidharan if (unlikely(!desc)) { 66784640e27SKaricheri, Muralidharan dev_err(netcp->ndev_dev, "failed to unmap Rx desc\n"); 66884640e27SKaricheri, Muralidharan return 0; 66984640e27SKaricheri, Muralidharan } 67084640e27SKaricheri, Muralidharan 67184640e27SKaricheri, Muralidharan get_pkt_info(&dma_buff, &buf_len, &dma_desc, desc); 67206324481SKaricheri, Muralidharan /* warning!!!! We are retrieving the virtual ptr in the sw_data 67306324481SKaricheri, Muralidharan * field as a 32bit value. Will not work on 64bit machines 67406324481SKaricheri, Muralidharan */ 67506324481SKaricheri, Muralidharan org_buf_ptr = (void *)GET_SW_DATA0(desc); 67606324481SKaricheri, Muralidharan org_buf_len = (int)GET_SW_DATA1(desc); 67784640e27SKaricheri, Muralidharan 67884640e27SKaricheri, Muralidharan if (unlikely(!org_buf_ptr)) { 67984640e27SKaricheri, Muralidharan dev_err(netcp->ndev_dev, "NULL bufptr in desc\n"); 68084640e27SKaricheri, Muralidharan goto free_desc; 68184640e27SKaricheri, Muralidharan } 68284640e27SKaricheri, Muralidharan 68384640e27SKaricheri, Muralidharan pkt_sz &= KNAV_DMA_DESC_PKT_LEN_MASK; 68484640e27SKaricheri, Muralidharan accum_sz = buf_len; 68584640e27SKaricheri, Muralidharan dma_unmap_single(netcp->dev, dma_buff, buf_len, DMA_FROM_DEVICE); 68684640e27SKaricheri, Muralidharan 68784640e27SKaricheri, Muralidharan /* Build a new sk_buff for the primary buffer */ 68884640e27SKaricheri, Muralidharan skb = build_skb(org_buf_ptr, org_buf_len); 68984640e27SKaricheri, Muralidharan if (unlikely(!skb)) { 69084640e27SKaricheri, Muralidharan dev_err(netcp->ndev_dev, "build_skb() failed\n"); 69184640e27SKaricheri, Muralidharan goto free_desc; 69284640e27SKaricheri, Muralidharan } 69384640e27SKaricheri, Muralidharan 69484640e27SKaricheri, Muralidharan /* update data, tail and len */ 69584640e27SKaricheri, Muralidharan skb_reserve(skb, NETCP_SOP_OFFSET); 69684640e27SKaricheri, Muralidharan __skb_put(skb, buf_len); 69784640e27SKaricheri, Muralidharan 69884640e27SKaricheri, Muralidharan /* Fill in the page fragment list */ 69984640e27SKaricheri, Muralidharan while (dma_desc) { 70084640e27SKaricheri, Muralidharan struct page *page; 70184640e27SKaricheri, Muralidharan 70284640e27SKaricheri, Muralidharan ndesc = knav_pool_desc_unmap(netcp->rx_pool, dma_desc, dma_sz); 70384640e27SKaricheri, Muralidharan if (unlikely(!ndesc)) { 70484640e27SKaricheri, Muralidharan dev_err(netcp->ndev_dev, "failed to unmap Rx desc\n"); 70584640e27SKaricheri, Muralidharan goto free_desc; 70684640e27SKaricheri, Muralidharan } 70784640e27SKaricheri, Muralidharan 70884640e27SKaricheri, Muralidharan get_pkt_info(&dma_buff, &buf_len, &dma_desc, ndesc); 70906324481SKaricheri, Muralidharan /* warning!!!! We are retrieving the virtual ptr in the sw_data 71006324481SKaricheri, Muralidharan * field as a 32bit value. Will not work on 64bit machines 71106324481SKaricheri, Muralidharan */ 7125a717843SRex Chang page = (struct page *)GET_SW_DATA0(ndesc); 71384640e27SKaricheri, Muralidharan 71484640e27SKaricheri, Muralidharan if (likely(dma_buff && buf_len && page)) { 71584640e27SKaricheri, Muralidharan dma_unmap_page(netcp->dev, dma_buff, PAGE_SIZE, 71684640e27SKaricheri, Muralidharan DMA_FROM_DEVICE); 71784640e27SKaricheri, Muralidharan } else { 71889907779SArnd Bergmann dev_err(netcp->ndev_dev, "Bad Rx desc dma_buff(%pad), len(%d), page(%p)\n", 71989907779SArnd Bergmann &dma_buff, buf_len, page); 72084640e27SKaricheri, Muralidharan goto free_desc; 72184640e27SKaricheri, Muralidharan } 72284640e27SKaricheri, Muralidharan 72384640e27SKaricheri, Muralidharan skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page, 72484640e27SKaricheri, Muralidharan offset_in_page(dma_buff), buf_len, PAGE_SIZE); 72584640e27SKaricheri, Muralidharan accum_sz += buf_len; 72684640e27SKaricheri, Muralidharan 72784640e27SKaricheri, Muralidharan /* Free the descriptor */ 72884640e27SKaricheri, Muralidharan knav_pool_desc_put(netcp->rx_pool, ndesc); 72984640e27SKaricheri, Muralidharan } 73084640e27SKaricheri, Muralidharan 73184640e27SKaricheri, Muralidharan /* check for packet len and warn */ 73284640e27SKaricheri, Muralidharan if (unlikely(pkt_sz != accum_sz)) 73384640e27SKaricheri, Muralidharan dev_dbg(netcp->ndev_dev, "mismatch in packet size(%d) & sum of fragments(%d)\n", 73484640e27SKaricheri, Muralidharan pkt_sz, accum_sz); 73584640e27SKaricheri, Muralidharan 7364cd85a61SKaricheri, Muralidharan /* Newer version of the Ethernet switch can trim the Ethernet FCS 7374cd85a61SKaricheri, Muralidharan * from the packet and is indicated in hw_cap. So trim it only for 7384cd85a61SKaricheri, Muralidharan * older h/w 7394cd85a61SKaricheri, Muralidharan */ 7404cd85a61SKaricheri, Muralidharan if (!(netcp->hw_cap & ETH_SW_CAN_REMOVE_ETH_FCS)) 74184640e27SKaricheri, Muralidharan __pskb_trim(skb, skb->len - ETH_FCS_LEN); 74284640e27SKaricheri, Muralidharan 74384640e27SKaricheri, Muralidharan /* Call each of the RX hooks */ 74484640e27SKaricheri, Muralidharan p_info.skb = skb; 7456246168bSWingMan Kwok skb->dev = netcp->ndev; 74684640e27SKaricheri, Muralidharan p_info.rxtstamp_complete = false; 74769d707d0SKaricheri, Muralidharan get_desc_info(&tmp, &p_info.eflags, desc); 74869d707d0SKaricheri, Muralidharan p_info.epib = desc->epib; 74969d707d0SKaricheri, Muralidharan p_info.psdata = (u32 __force *)desc->psdata; 75069d707d0SKaricheri, Muralidharan p_info.eflags = ((p_info.eflags >> KNAV_DMA_DESC_EFLAGS_SHIFT) & 75169d707d0SKaricheri, Muralidharan KNAV_DMA_DESC_EFLAGS_MASK); 75284640e27SKaricheri, Muralidharan list_for_each_entry(rx_hook, &netcp->rxhook_list_head, list) { 75384640e27SKaricheri, Muralidharan int ret; 75484640e27SKaricheri, Muralidharan 75584640e27SKaricheri, Muralidharan ret = rx_hook->hook_rtn(rx_hook->order, rx_hook->hook_data, 75684640e27SKaricheri, Muralidharan &p_info); 75784640e27SKaricheri, Muralidharan if (unlikely(ret)) { 75884640e27SKaricheri, Muralidharan dev_err(netcp->ndev_dev, "RX hook %d failed: %d\n", 75984640e27SKaricheri, Muralidharan rx_hook->order, ret); 76069d707d0SKaricheri, Muralidharan /* Free the primary descriptor */ 7616a8162e9SMichael Scherban rx_stats->rx_dropped++; 76269d707d0SKaricheri, Muralidharan knav_pool_desc_put(netcp->rx_pool, desc); 76384640e27SKaricheri, Muralidharan dev_kfree_skb(skb); 76484640e27SKaricheri, Muralidharan return 0; 76584640e27SKaricheri, Muralidharan } 76684640e27SKaricheri, Muralidharan } 76769d707d0SKaricheri, Muralidharan /* Free the primary descriptor */ 76869d707d0SKaricheri, Muralidharan knav_pool_desc_put(netcp->rx_pool, desc); 76984640e27SKaricheri, Muralidharan 7706a8162e9SMichael Scherban u64_stats_update_begin(&rx_stats->syncp_rx); 7716a8162e9SMichael Scherban rx_stats->rx_packets++; 7726a8162e9SMichael Scherban rx_stats->rx_bytes += skb->len; 7736a8162e9SMichael Scherban u64_stats_update_end(&rx_stats->syncp_rx); 77484640e27SKaricheri, Muralidharan 77584640e27SKaricheri, Muralidharan /* push skb up the stack */ 77684640e27SKaricheri, Muralidharan skb->protocol = eth_type_trans(skb, netcp->ndev); 77784640e27SKaricheri, Muralidharan netif_receive_skb(skb); 77884640e27SKaricheri, Muralidharan return 0; 77984640e27SKaricheri, Muralidharan 78084640e27SKaricheri, Muralidharan free_desc: 78184640e27SKaricheri, Muralidharan netcp_free_rx_desc_chain(netcp, desc); 7826a8162e9SMichael Scherban rx_stats->rx_errors++; 78384640e27SKaricheri, Muralidharan return 0; 78484640e27SKaricheri, Muralidharan } 78584640e27SKaricheri, Muralidharan 78684640e27SKaricheri, Muralidharan static int netcp_process_rx_packets(struct netcp_intf *netcp, 78784640e27SKaricheri, Muralidharan unsigned int budget) 78884640e27SKaricheri, Muralidharan { 78984640e27SKaricheri, Muralidharan int i; 79084640e27SKaricheri, Muralidharan 79184640e27SKaricheri, Muralidharan for (i = 0; (i < budget) && !netcp_process_one_rx_packet(netcp); i++) 79284640e27SKaricheri, Muralidharan ; 79384640e27SKaricheri, Muralidharan return i; 79484640e27SKaricheri, Muralidharan } 79584640e27SKaricheri, Muralidharan 79684640e27SKaricheri, Muralidharan /* Release descriptors and attached buffers from Rx FDQ */ 79784640e27SKaricheri, Muralidharan static void netcp_free_rx_buf(struct netcp_intf *netcp, int fdq) 79884640e27SKaricheri, Muralidharan { 79984640e27SKaricheri, Muralidharan struct knav_dma_desc *desc; 80084640e27SKaricheri, Muralidharan unsigned int buf_len, dma_sz; 80184640e27SKaricheri, Muralidharan dma_addr_t dma; 80284640e27SKaricheri, Muralidharan void *buf_ptr; 80384640e27SKaricheri, Muralidharan 80484640e27SKaricheri, Muralidharan /* Allocate descriptor */ 80584640e27SKaricheri, Muralidharan while ((dma = knav_queue_pop(netcp->rx_fdq[fdq], &dma_sz))) { 80684640e27SKaricheri, Muralidharan desc = knav_pool_desc_unmap(netcp->rx_pool, dma, dma_sz); 80784640e27SKaricheri, Muralidharan if (unlikely(!desc)) { 80884640e27SKaricheri, Muralidharan dev_err(netcp->ndev_dev, "failed to unmap Rx desc\n"); 80984640e27SKaricheri, Muralidharan continue; 81084640e27SKaricheri, Muralidharan } 81184640e27SKaricheri, Muralidharan 81284640e27SKaricheri, Muralidharan get_org_pkt_info(&dma, &buf_len, desc); 81306324481SKaricheri, Muralidharan /* warning!!!! We are retrieving the virtual ptr in the sw_data 81406324481SKaricheri, Muralidharan * field as a 32bit value. Will not work on 64bit machines 81506324481SKaricheri, Muralidharan */ 81606324481SKaricheri, Muralidharan buf_ptr = (void *)GET_SW_DATA0(desc); 81784640e27SKaricheri, Muralidharan 81884640e27SKaricheri, Muralidharan if (unlikely(!dma)) { 81984640e27SKaricheri, Muralidharan dev_err(netcp->ndev_dev, "NULL orig_buff in desc\n"); 82084640e27SKaricheri, Muralidharan knav_pool_desc_put(netcp->rx_pool, desc); 82184640e27SKaricheri, Muralidharan continue; 82284640e27SKaricheri, Muralidharan } 82384640e27SKaricheri, Muralidharan 82484640e27SKaricheri, Muralidharan if (unlikely(!buf_ptr)) { 82584640e27SKaricheri, Muralidharan dev_err(netcp->ndev_dev, "NULL bufptr in desc\n"); 82684640e27SKaricheri, Muralidharan knav_pool_desc_put(netcp->rx_pool, desc); 82784640e27SKaricheri, Muralidharan continue; 82884640e27SKaricheri, Muralidharan } 82984640e27SKaricheri, Muralidharan 83084640e27SKaricheri, Muralidharan if (fdq == 0) { 83184640e27SKaricheri, Muralidharan dma_unmap_single(netcp->dev, dma, buf_len, 83284640e27SKaricheri, Muralidharan DMA_FROM_DEVICE); 83384640e27SKaricheri, Muralidharan netcp_frag_free((buf_len <= PAGE_SIZE), buf_ptr); 83484640e27SKaricheri, Muralidharan } else { 83584640e27SKaricheri, Muralidharan dma_unmap_page(netcp->dev, dma, buf_len, 83684640e27SKaricheri, Muralidharan DMA_FROM_DEVICE); 83784640e27SKaricheri, Muralidharan __free_page(buf_ptr); 83884640e27SKaricheri, Muralidharan } 83984640e27SKaricheri, Muralidharan 84084640e27SKaricheri, Muralidharan knav_pool_desc_put(netcp->rx_pool, desc); 84184640e27SKaricheri, Muralidharan } 84284640e27SKaricheri, Muralidharan } 84384640e27SKaricheri, Muralidharan 84484640e27SKaricheri, Muralidharan static void netcp_rxpool_free(struct netcp_intf *netcp) 84584640e27SKaricheri, Muralidharan { 84684640e27SKaricheri, Muralidharan int i; 84784640e27SKaricheri, Muralidharan 84884640e27SKaricheri, Muralidharan for (i = 0; i < KNAV_DMA_FDQ_PER_CHAN && 84984640e27SKaricheri, Muralidharan !IS_ERR_OR_NULL(netcp->rx_fdq[i]); i++) 85084640e27SKaricheri, Muralidharan netcp_free_rx_buf(netcp, i); 85184640e27SKaricheri, Muralidharan 85284640e27SKaricheri, Muralidharan if (knav_pool_count(netcp->rx_pool) != netcp->rx_pool_size) 85384640e27SKaricheri, Muralidharan dev_err(netcp->ndev_dev, "Lost Rx (%d) descriptors\n", 85484640e27SKaricheri, Muralidharan netcp->rx_pool_size - knav_pool_count(netcp->rx_pool)); 85584640e27SKaricheri, Muralidharan 85684640e27SKaricheri, Muralidharan knav_pool_destroy(netcp->rx_pool); 85784640e27SKaricheri, Muralidharan netcp->rx_pool = NULL; 85884640e27SKaricheri, Muralidharan } 85984640e27SKaricheri, Muralidharan 860e558b1fbSKaricheri, Muralidharan static int netcp_allocate_rx_buf(struct netcp_intf *netcp, int fdq) 86184640e27SKaricheri, Muralidharan { 86284640e27SKaricheri, Muralidharan struct knav_dma_desc *hwdesc; 86384640e27SKaricheri, Muralidharan unsigned int buf_len, dma_sz; 86484640e27SKaricheri, Muralidharan u32 desc_info, pkt_info; 86584640e27SKaricheri, Muralidharan struct page *page; 86684640e27SKaricheri, Muralidharan dma_addr_t dma; 86784640e27SKaricheri, Muralidharan void *bufptr; 868b1cb86aeSKaricheri, Muralidharan u32 sw_data[2]; 86984640e27SKaricheri, Muralidharan 87084640e27SKaricheri, Muralidharan /* Allocate descriptor */ 87184640e27SKaricheri, Muralidharan hwdesc = knav_pool_desc_get(netcp->rx_pool); 87284640e27SKaricheri, Muralidharan if (IS_ERR_OR_NULL(hwdesc)) { 87384640e27SKaricheri, Muralidharan dev_dbg(netcp->ndev_dev, "out of rx pool desc\n"); 874e558b1fbSKaricheri, Muralidharan return -ENOMEM; 87584640e27SKaricheri, Muralidharan } 87684640e27SKaricheri, Muralidharan 87784640e27SKaricheri, Muralidharan if (likely(fdq == 0)) { 87884640e27SKaricheri, Muralidharan unsigned int primary_buf_len; 87984640e27SKaricheri, Muralidharan /* Allocate a primary receive queue entry */ 880866b8b18SWingMan Kwok buf_len = NETCP_PACKET_SIZE + NETCP_SOP_OFFSET; 88184640e27SKaricheri, Muralidharan primary_buf_len = SKB_DATA_ALIGN(buf_len) + 88284640e27SKaricheri, Muralidharan SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 88384640e27SKaricheri, Muralidharan 88484640e27SKaricheri, Muralidharan bufptr = netdev_alloc_frag(primary_buf_len); 885b1cb86aeSKaricheri, Muralidharan sw_data[1] = primary_buf_len; 88684640e27SKaricheri, Muralidharan 88784640e27SKaricheri, Muralidharan if (unlikely(!bufptr)) { 888866b8b18SWingMan Kwok dev_warn_ratelimited(netcp->ndev_dev, 889866b8b18SWingMan Kwok "Primary RX buffer alloc failed\n"); 89084640e27SKaricheri, Muralidharan goto fail; 89184640e27SKaricheri, Muralidharan } 89284640e27SKaricheri, Muralidharan dma = dma_map_single(netcp->dev, bufptr, buf_len, 89384640e27SKaricheri, Muralidharan DMA_TO_DEVICE); 894866b8b18SWingMan Kwok if (unlikely(dma_mapping_error(netcp->dev, dma))) 895866b8b18SWingMan Kwok goto fail; 896866b8b18SWingMan Kwok 89706324481SKaricheri, Muralidharan /* warning!!!! We are saving the virtual ptr in the sw_data 89806324481SKaricheri, Muralidharan * field as a 32bit value. Will not work on 64bit machines 89906324481SKaricheri, Muralidharan */ 900b1cb86aeSKaricheri, Muralidharan sw_data[0] = (u32)bufptr; 90184640e27SKaricheri, Muralidharan } else { 90284640e27SKaricheri, Muralidharan /* Allocate a secondary receive queue entry */ 903453f85d4SMel Gorman page = alloc_page(GFP_ATOMIC | GFP_DMA); 90484640e27SKaricheri, Muralidharan if (unlikely(!page)) { 90584640e27SKaricheri, Muralidharan dev_warn_ratelimited(netcp->ndev_dev, "Secondary page alloc failed\n"); 90684640e27SKaricheri, Muralidharan goto fail; 90784640e27SKaricheri, Muralidharan } 90884640e27SKaricheri, Muralidharan buf_len = PAGE_SIZE; 90984640e27SKaricheri, Muralidharan dma = dma_map_page(netcp->dev, page, 0, buf_len, DMA_TO_DEVICE); 91006324481SKaricheri, Muralidharan /* warning!!!! We are saving the virtual ptr in the sw_data 91106324481SKaricheri, Muralidharan * field as a 32bit value. Will not work on 64bit machines 91206324481SKaricheri, Muralidharan */ 913b1cb86aeSKaricheri, Muralidharan sw_data[0] = (u32)page; 914b1cb86aeSKaricheri, Muralidharan sw_data[1] = 0; 91584640e27SKaricheri, Muralidharan } 91684640e27SKaricheri, Muralidharan 91784640e27SKaricheri, Muralidharan desc_info = KNAV_DMA_DESC_PS_INFO_IN_DESC; 91884640e27SKaricheri, Muralidharan desc_info |= buf_len & KNAV_DMA_DESC_PKT_LEN_MASK; 91984640e27SKaricheri, Muralidharan pkt_info = KNAV_DMA_DESC_HAS_EPIB; 92084640e27SKaricheri, Muralidharan pkt_info |= KNAV_DMA_NUM_PS_WORDS << KNAV_DMA_DESC_PSLEN_SHIFT; 92184640e27SKaricheri, Muralidharan pkt_info |= (netcp->rx_queue_id & KNAV_DMA_DESC_RETQ_MASK) << 92284640e27SKaricheri, Muralidharan KNAV_DMA_DESC_RETQ_SHIFT; 92384640e27SKaricheri, Muralidharan set_org_pkt_info(dma, buf_len, hwdesc); 92406324481SKaricheri, Muralidharan SET_SW_DATA0(sw_data[0], hwdesc); 92506324481SKaricheri, Muralidharan SET_SW_DATA1(sw_data[1], hwdesc); 92684640e27SKaricheri, Muralidharan set_desc_info(desc_info, pkt_info, hwdesc); 92784640e27SKaricheri, Muralidharan 92884640e27SKaricheri, Muralidharan /* Push to FDQs */ 92984640e27SKaricheri, Muralidharan knav_pool_desc_map(netcp->rx_pool, hwdesc, sizeof(*hwdesc), &dma, 93084640e27SKaricheri, Muralidharan &dma_sz); 93184640e27SKaricheri, Muralidharan knav_queue_push(netcp->rx_fdq[fdq], dma, sizeof(*hwdesc), 0); 932e558b1fbSKaricheri, Muralidharan return 0; 93384640e27SKaricheri, Muralidharan 93484640e27SKaricheri, Muralidharan fail: 93584640e27SKaricheri, Muralidharan knav_pool_desc_put(netcp->rx_pool, hwdesc); 936e558b1fbSKaricheri, Muralidharan return -ENOMEM; 93784640e27SKaricheri, Muralidharan } 93884640e27SKaricheri, Muralidharan 93984640e27SKaricheri, Muralidharan /* Refill Rx FDQ with descriptors & attached buffers */ 94084640e27SKaricheri, Muralidharan static void netcp_rxpool_refill(struct netcp_intf *netcp) 94184640e27SKaricheri, Muralidharan { 94284640e27SKaricheri, Muralidharan u32 fdq_deficit[KNAV_DMA_FDQ_PER_CHAN] = {0}; 943e558b1fbSKaricheri, Muralidharan int i, ret = 0; 94484640e27SKaricheri, Muralidharan 94584640e27SKaricheri, Muralidharan /* Calculate the FDQ deficit and refill */ 94684640e27SKaricheri, Muralidharan for (i = 0; i < KNAV_DMA_FDQ_PER_CHAN && netcp->rx_fdq[i]; i++) { 94784640e27SKaricheri, Muralidharan fdq_deficit[i] = netcp->rx_queue_depths[i] - 94884640e27SKaricheri, Muralidharan knav_queue_get_count(netcp->rx_fdq[i]); 94984640e27SKaricheri, Muralidharan 950e558b1fbSKaricheri, Muralidharan while (fdq_deficit[i]-- && !ret) 951e558b1fbSKaricheri, Muralidharan ret = netcp_allocate_rx_buf(netcp, i); 95284640e27SKaricheri, Muralidharan } /* end for fdqs */ 95384640e27SKaricheri, Muralidharan } 95484640e27SKaricheri, Muralidharan 95584640e27SKaricheri, Muralidharan /* NAPI poll */ 95684640e27SKaricheri, Muralidharan static int netcp_rx_poll(struct napi_struct *napi, int budget) 95784640e27SKaricheri, Muralidharan { 95884640e27SKaricheri, Muralidharan struct netcp_intf *netcp = container_of(napi, struct netcp_intf, 95984640e27SKaricheri, Muralidharan rx_napi); 96084640e27SKaricheri, Muralidharan unsigned int packets; 96184640e27SKaricheri, Muralidharan 96284640e27SKaricheri, Muralidharan packets = netcp_process_rx_packets(netcp, budget); 96384640e27SKaricheri, Muralidharan 96499f8ef5dSKaricheri, Muralidharan netcp_rxpool_refill(netcp); 96584640e27SKaricheri, Muralidharan if (packets < budget) { 9666ad20165SEric Dumazet napi_complete_done(&netcp->rx_napi, packets); 96784640e27SKaricheri, Muralidharan knav_queue_enable_notify(netcp->rx_queue); 96884640e27SKaricheri, Muralidharan } 96984640e27SKaricheri, Muralidharan 97084640e27SKaricheri, Muralidharan return packets; 97184640e27SKaricheri, Muralidharan } 97284640e27SKaricheri, Muralidharan 97384640e27SKaricheri, Muralidharan static void netcp_rx_notify(void *arg) 97484640e27SKaricheri, Muralidharan { 97584640e27SKaricheri, Muralidharan struct netcp_intf *netcp = arg; 97684640e27SKaricheri, Muralidharan 97784640e27SKaricheri, Muralidharan knav_queue_disable_notify(netcp->rx_queue); 97884640e27SKaricheri, Muralidharan napi_schedule(&netcp->rx_napi); 97984640e27SKaricheri, Muralidharan } 98084640e27SKaricheri, Muralidharan 98184640e27SKaricheri, Muralidharan static void netcp_free_tx_desc_chain(struct netcp_intf *netcp, 98284640e27SKaricheri, Muralidharan struct knav_dma_desc *desc, 98384640e27SKaricheri, Muralidharan unsigned int desc_sz) 98484640e27SKaricheri, Muralidharan { 98584640e27SKaricheri, Muralidharan struct knav_dma_desc *ndesc = desc; 98684640e27SKaricheri, Muralidharan dma_addr_t dma_desc, dma_buf; 98784640e27SKaricheri, Muralidharan unsigned int buf_len; 98884640e27SKaricheri, Muralidharan 98984640e27SKaricheri, Muralidharan while (ndesc) { 99084640e27SKaricheri, Muralidharan get_pkt_info(&dma_buf, &buf_len, &dma_desc, ndesc); 99184640e27SKaricheri, Muralidharan 99284640e27SKaricheri, Muralidharan if (dma_buf && buf_len) 99384640e27SKaricheri, Muralidharan dma_unmap_single(netcp->dev, dma_buf, buf_len, 99484640e27SKaricheri, Muralidharan DMA_TO_DEVICE); 99584640e27SKaricheri, Muralidharan else 99689907779SArnd Bergmann dev_warn(netcp->ndev_dev, "bad Tx desc buf(%pad), len(%d)\n", 99789907779SArnd Bergmann &dma_buf, buf_len); 99884640e27SKaricheri, Muralidharan 99984640e27SKaricheri, Muralidharan knav_pool_desc_put(netcp->tx_pool, ndesc); 100084640e27SKaricheri, Muralidharan ndesc = NULL; 100184640e27SKaricheri, Muralidharan if (dma_desc) { 100284640e27SKaricheri, Muralidharan ndesc = knav_pool_desc_unmap(netcp->tx_pool, dma_desc, 100384640e27SKaricheri, Muralidharan desc_sz); 100484640e27SKaricheri, Muralidharan if (!ndesc) 100584640e27SKaricheri, Muralidharan dev_err(netcp->ndev_dev, "failed to unmap Tx desc\n"); 100684640e27SKaricheri, Muralidharan } 100784640e27SKaricheri, Muralidharan } 100884640e27SKaricheri, Muralidharan } 100984640e27SKaricheri, Muralidharan 101084640e27SKaricheri, Muralidharan static int netcp_process_tx_compl_packets(struct netcp_intf *netcp, 101184640e27SKaricheri, Muralidharan unsigned int budget) 101284640e27SKaricheri, Muralidharan { 10136a8162e9SMichael Scherban struct netcp_stats *tx_stats = &netcp->stats; 101484640e27SKaricheri, Muralidharan struct knav_dma_desc *desc; 10156246168bSWingMan Kwok struct netcp_tx_cb *tx_cb; 101684640e27SKaricheri, Muralidharan struct sk_buff *skb; 101784640e27SKaricheri, Muralidharan unsigned int dma_sz; 101884640e27SKaricheri, Muralidharan dma_addr_t dma; 101984640e27SKaricheri, Muralidharan int pkts = 0; 102084640e27SKaricheri, Muralidharan 102184640e27SKaricheri, Muralidharan while (budget--) { 102284640e27SKaricheri, Muralidharan dma = knav_queue_pop(netcp->tx_compl_q, &dma_sz); 102384640e27SKaricheri, Muralidharan if (!dma) 102484640e27SKaricheri, Muralidharan break; 102584640e27SKaricheri, Muralidharan desc = knav_pool_desc_unmap(netcp->tx_pool, dma, dma_sz); 102684640e27SKaricheri, Muralidharan if (unlikely(!desc)) { 102784640e27SKaricheri, Muralidharan dev_err(netcp->ndev_dev, "failed to unmap Tx desc\n"); 10286a8162e9SMichael Scherban tx_stats->tx_errors++; 102984640e27SKaricheri, Muralidharan continue; 103084640e27SKaricheri, Muralidharan } 103184640e27SKaricheri, Muralidharan 103206324481SKaricheri, Muralidharan /* warning!!!! We are retrieving the virtual ptr in the sw_data 103306324481SKaricheri, Muralidharan * field as a 32bit value. Will not work on 64bit machines 103406324481SKaricheri, Muralidharan */ 103506324481SKaricheri, Muralidharan skb = (struct sk_buff *)GET_SW_DATA0(desc); 103684640e27SKaricheri, Muralidharan netcp_free_tx_desc_chain(netcp, desc, dma_sz); 103784640e27SKaricheri, Muralidharan if (!skb) { 103884640e27SKaricheri, Muralidharan dev_err(netcp->ndev_dev, "No skb in Tx desc\n"); 10396a8162e9SMichael Scherban tx_stats->tx_errors++; 104084640e27SKaricheri, Muralidharan continue; 104184640e27SKaricheri, Muralidharan } 104284640e27SKaricheri, Muralidharan 10436246168bSWingMan Kwok tx_cb = (struct netcp_tx_cb *)skb->cb; 10446246168bSWingMan Kwok if (tx_cb->txtstamp) 10456246168bSWingMan Kwok tx_cb->txtstamp(tx_cb->ts_context, skb); 10466246168bSWingMan Kwok 104784640e27SKaricheri, Muralidharan if (netif_subqueue_stopped(netcp->ndev, skb) && 104884640e27SKaricheri, Muralidharan netif_running(netcp->ndev) && 104984640e27SKaricheri, Muralidharan (knav_pool_count(netcp->tx_pool) > 105084640e27SKaricheri, Muralidharan netcp->tx_resume_threshold)) { 105184640e27SKaricheri, Muralidharan u16 subqueue = skb_get_queue_mapping(skb); 105284640e27SKaricheri, Muralidharan 105384640e27SKaricheri, Muralidharan netif_wake_subqueue(netcp->ndev, subqueue); 105484640e27SKaricheri, Muralidharan } 105584640e27SKaricheri, Muralidharan 10566a8162e9SMichael Scherban u64_stats_update_begin(&tx_stats->syncp_tx); 10576a8162e9SMichael Scherban tx_stats->tx_packets++; 10586a8162e9SMichael Scherban tx_stats->tx_bytes += skb->len; 10596a8162e9SMichael Scherban u64_stats_update_end(&tx_stats->syncp_tx); 106084640e27SKaricheri, Muralidharan dev_kfree_skb(skb); 106184640e27SKaricheri, Muralidharan pkts++; 106284640e27SKaricheri, Muralidharan } 106384640e27SKaricheri, Muralidharan return pkts; 106484640e27SKaricheri, Muralidharan } 106584640e27SKaricheri, Muralidharan 106684640e27SKaricheri, Muralidharan static int netcp_tx_poll(struct napi_struct *napi, int budget) 106784640e27SKaricheri, Muralidharan { 106884640e27SKaricheri, Muralidharan int packets; 106984640e27SKaricheri, Muralidharan struct netcp_intf *netcp = container_of(napi, struct netcp_intf, 107084640e27SKaricheri, Muralidharan tx_napi); 107184640e27SKaricheri, Muralidharan 107284640e27SKaricheri, Muralidharan packets = netcp_process_tx_compl_packets(netcp, budget); 107384640e27SKaricheri, Muralidharan if (packets < budget) { 107484640e27SKaricheri, Muralidharan napi_complete(&netcp->tx_napi); 107584640e27SKaricheri, Muralidharan knav_queue_enable_notify(netcp->tx_compl_q); 107684640e27SKaricheri, Muralidharan } 107784640e27SKaricheri, Muralidharan 107884640e27SKaricheri, Muralidharan return packets; 107984640e27SKaricheri, Muralidharan } 108084640e27SKaricheri, Muralidharan 108184640e27SKaricheri, Muralidharan static void netcp_tx_notify(void *arg) 108284640e27SKaricheri, Muralidharan { 108384640e27SKaricheri, Muralidharan struct netcp_intf *netcp = arg; 108484640e27SKaricheri, Muralidharan 108584640e27SKaricheri, Muralidharan knav_queue_disable_notify(netcp->tx_compl_q); 108684640e27SKaricheri, Muralidharan napi_schedule(&netcp->tx_napi); 108784640e27SKaricheri, Muralidharan } 108884640e27SKaricheri, Muralidharan 108984640e27SKaricheri, Muralidharan static struct knav_dma_desc* 109084640e27SKaricheri, Muralidharan netcp_tx_map_skb(struct sk_buff *skb, struct netcp_intf *netcp) 109184640e27SKaricheri, Muralidharan { 109284640e27SKaricheri, Muralidharan struct knav_dma_desc *desc, *ndesc, *pdesc; 109384640e27SKaricheri, Muralidharan unsigned int pkt_len = skb_headlen(skb); 109484640e27SKaricheri, Muralidharan struct device *dev = netcp->dev; 109584640e27SKaricheri, Muralidharan dma_addr_t dma_addr; 109684640e27SKaricheri, Muralidharan unsigned int dma_sz; 109784640e27SKaricheri, Muralidharan int i; 109884640e27SKaricheri, Muralidharan 109984640e27SKaricheri, Muralidharan /* Map the linear buffer */ 110084640e27SKaricheri, Muralidharan dma_addr = dma_map_single(dev, skb->data, pkt_len, DMA_TO_DEVICE); 1101866b8b18SWingMan Kwok if (unlikely(dma_mapping_error(dev, dma_addr))) { 110284640e27SKaricheri, Muralidharan dev_err(netcp->ndev_dev, "Failed to map skb buffer\n"); 110384640e27SKaricheri, Muralidharan return NULL; 110484640e27SKaricheri, Muralidharan } 110584640e27SKaricheri, Muralidharan 110684640e27SKaricheri, Muralidharan desc = knav_pool_desc_get(netcp->tx_pool); 1107bf69a3b6SViresh Kumar if (IS_ERR_OR_NULL(desc)) { 110884640e27SKaricheri, Muralidharan dev_err(netcp->ndev_dev, "out of TX desc\n"); 110984640e27SKaricheri, Muralidharan dma_unmap_single(dev, dma_addr, pkt_len, DMA_TO_DEVICE); 111084640e27SKaricheri, Muralidharan return NULL; 111184640e27SKaricheri, Muralidharan } 111284640e27SKaricheri, Muralidharan 111384640e27SKaricheri, Muralidharan set_pkt_info(dma_addr, pkt_len, 0, desc); 111484640e27SKaricheri, Muralidharan if (skb_is_nonlinear(skb)) { 111584640e27SKaricheri, Muralidharan prefetchw(skb_shinfo(skb)); 111684640e27SKaricheri, Muralidharan } else { 111784640e27SKaricheri, Muralidharan desc->next_desc = 0; 111884640e27SKaricheri, Muralidharan goto upd_pkt_len; 111984640e27SKaricheri, Muralidharan } 112084640e27SKaricheri, Muralidharan 112184640e27SKaricheri, Muralidharan pdesc = desc; 112284640e27SKaricheri, Muralidharan 112384640e27SKaricheri, Muralidharan /* Handle the case where skb is fragmented in pages */ 112484640e27SKaricheri, Muralidharan for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 112584640e27SKaricheri, Muralidharan skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 112684640e27SKaricheri, Muralidharan struct page *page = skb_frag_page(frag); 112784640e27SKaricheri, Muralidharan u32 page_offset = frag->page_offset; 112884640e27SKaricheri, Muralidharan u32 buf_len = skb_frag_size(frag); 112984640e27SKaricheri, Muralidharan dma_addr_t desc_dma; 113089907779SArnd Bergmann u32 desc_dma_32; 113184640e27SKaricheri, Muralidharan 113284640e27SKaricheri, Muralidharan dma_addr = dma_map_page(dev, page, page_offset, buf_len, 113384640e27SKaricheri, Muralidharan DMA_TO_DEVICE); 113484640e27SKaricheri, Muralidharan if (unlikely(!dma_addr)) { 113584640e27SKaricheri, Muralidharan dev_err(netcp->ndev_dev, "Failed to map skb page\n"); 113684640e27SKaricheri, Muralidharan goto free_descs; 113784640e27SKaricheri, Muralidharan } 113884640e27SKaricheri, Muralidharan 113984640e27SKaricheri, Muralidharan ndesc = knav_pool_desc_get(netcp->tx_pool); 1140bf69a3b6SViresh Kumar if (IS_ERR_OR_NULL(ndesc)) { 114184640e27SKaricheri, Muralidharan dev_err(netcp->ndev_dev, "out of TX desc for frags\n"); 114284640e27SKaricheri, Muralidharan dma_unmap_page(dev, dma_addr, buf_len, DMA_TO_DEVICE); 114384640e27SKaricheri, Muralidharan goto free_descs; 114484640e27SKaricheri, Muralidharan } 114584640e27SKaricheri, Muralidharan 114689907779SArnd Bergmann desc_dma = knav_pool_desc_virt_to_dma(netcp->tx_pool, ndesc); 114784640e27SKaricheri, Muralidharan set_pkt_info(dma_addr, buf_len, 0, ndesc); 114889907779SArnd Bergmann desc_dma_32 = (u32)desc_dma; 114989907779SArnd Bergmann set_words(&desc_dma_32, 1, &pdesc->next_desc); 115084640e27SKaricheri, Muralidharan pkt_len += buf_len; 115184640e27SKaricheri, Muralidharan if (pdesc != desc) 115284640e27SKaricheri, Muralidharan knav_pool_desc_map(netcp->tx_pool, pdesc, 115384640e27SKaricheri, Muralidharan sizeof(*pdesc), &desc_dma, &dma_sz); 115484640e27SKaricheri, Muralidharan pdesc = ndesc; 115584640e27SKaricheri, Muralidharan } 115684640e27SKaricheri, Muralidharan if (pdesc != desc) 115784640e27SKaricheri, Muralidharan knav_pool_desc_map(netcp->tx_pool, pdesc, sizeof(*pdesc), 115884640e27SKaricheri, Muralidharan &dma_addr, &dma_sz); 115984640e27SKaricheri, Muralidharan 116084640e27SKaricheri, Muralidharan /* frag list based linkage is not supported for now. */ 116184640e27SKaricheri, Muralidharan if (skb_shinfo(skb)->frag_list) { 116284640e27SKaricheri, Muralidharan dev_err_ratelimited(netcp->ndev_dev, "NETIF_F_FRAGLIST not supported\n"); 116384640e27SKaricheri, Muralidharan goto free_descs; 116484640e27SKaricheri, Muralidharan } 116584640e27SKaricheri, Muralidharan 116684640e27SKaricheri, Muralidharan upd_pkt_len: 116784640e27SKaricheri, Muralidharan WARN_ON(pkt_len != skb->len); 116884640e27SKaricheri, Muralidharan 116984640e27SKaricheri, Muralidharan pkt_len &= KNAV_DMA_DESC_PKT_LEN_MASK; 117084640e27SKaricheri, Muralidharan set_words(&pkt_len, 1, &desc->desc_info); 117184640e27SKaricheri, Muralidharan return desc; 117284640e27SKaricheri, Muralidharan 117384640e27SKaricheri, Muralidharan free_descs: 117484640e27SKaricheri, Muralidharan netcp_free_tx_desc_chain(netcp, desc, sizeof(*desc)); 117584640e27SKaricheri, Muralidharan return NULL; 117684640e27SKaricheri, Muralidharan } 117784640e27SKaricheri, Muralidharan 117884640e27SKaricheri, Muralidharan static int netcp_tx_submit_skb(struct netcp_intf *netcp, 117984640e27SKaricheri, Muralidharan struct sk_buff *skb, 118084640e27SKaricheri, Muralidharan struct knav_dma_desc *desc) 118184640e27SKaricheri, Muralidharan { 118284640e27SKaricheri, Muralidharan struct netcp_tx_pipe *tx_pipe = NULL; 118384640e27SKaricheri, Muralidharan struct netcp_hook_list *tx_hook; 118484640e27SKaricheri, Muralidharan struct netcp_packet p_info; 11856246168bSWingMan Kwok struct netcp_tx_cb *tx_cb; 118684640e27SKaricheri, Muralidharan unsigned int dma_sz; 118784640e27SKaricheri, Muralidharan dma_addr_t dma; 1188e170f409SKaricheri, Muralidharan u32 tmp = 0; 118984640e27SKaricheri, Muralidharan int ret = 0; 119084640e27SKaricheri, Muralidharan 119184640e27SKaricheri, Muralidharan p_info.netcp = netcp; 119284640e27SKaricheri, Muralidharan p_info.skb = skb; 119384640e27SKaricheri, Muralidharan p_info.tx_pipe = NULL; 119484640e27SKaricheri, Muralidharan p_info.psdata_len = 0; 119584640e27SKaricheri, Muralidharan p_info.ts_context = NULL; 11966246168bSWingMan Kwok p_info.txtstamp = NULL; 119784640e27SKaricheri, Muralidharan p_info.epib = desc->epib; 11989dd2d6c5SArnd Bergmann p_info.psdata = (u32 __force *)desc->psdata; 11999dd2d6c5SArnd Bergmann memset(p_info.epib, 0, KNAV_DMA_NUM_EPIB_WORDS * sizeof(__le32)); 120084640e27SKaricheri, Muralidharan 120184640e27SKaricheri, Muralidharan /* Find out where to inject the packet for transmission */ 120284640e27SKaricheri, Muralidharan list_for_each_entry(tx_hook, &netcp->txhook_list_head, list) { 120384640e27SKaricheri, Muralidharan ret = tx_hook->hook_rtn(tx_hook->order, tx_hook->hook_data, 120484640e27SKaricheri, Muralidharan &p_info); 120584640e27SKaricheri, Muralidharan if (unlikely(ret != 0)) { 120684640e27SKaricheri, Muralidharan dev_err(netcp->ndev_dev, "TX hook %d rejected the packet with reason(%d)\n", 120784640e27SKaricheri, Muralidharan tx_hook->order, ret); 120884640e27SKaricheri, Muralidharan ret = (ret < 0) ? ret : NETDEV_TX_OK; 120984640e27SKaricheri, Muralidharan goto out; 121084640e27SKaricheri, Muralidharan } 121184640e27SKaricheri, Muralidharan } 121284640e27SKaricheri, Muralidharan 121384640e27SKaricheri, Muralidharan /* Make sure some TX hook claimed the packet */ 121484640e27SKaricheri, Muralidharan tx_pipe = p_info.tx_pipe; 121584640e27SKaricheri, Muralidharan if (!tx_pipe) { 121684640e27SKaricheri, Muralidharan dev_err(netcp->ndev_dev, "No TX hook claimed the packet!\n"); 121784640e27SKaricheri, Muralidharan ret = -ENXIO; 121884640e27SKaricheri, Muralidharan goto out; 121984640e27SKaricheri, Muralidharan } 122084640e27SKaricheri, Muralidharan 12216246168bSWingMan Kwok tx_cb = (struct netcp_tx_cb *)skb->cb; 12226246168bSWingMan Kwok tx_cb->ts_context = p_info.ts_context; 12236246168bSWingMan Kwok tx_cb->txtstamp = p_info.txtstamp; 12246246168bSWingMan Kwok 122584640e27SKaricheri, Muralidharan /* update descriptor */ 122684640e27SKaricheri, Muralidharan if (p_info.psdata_len) { 12279dd2d6c5SArnd Bergmann /* psdata points to both native-endian and device-endian data */ 12289dd2d6c5SArnd Bergmann __le32 *psdata = (void __force *)p_info.psdata; 122984640e27SKaricheri, Muralidharan 1230aa255101SKaricheri, Muralidharan set_words((u32 *)psdata + 1231aa255101SKaricheri, Muralidharan (KNAV_DMA_NUM_PS_WORDS - p_info.psdata_len), 1232aa255101SKaricheri, Muralidharan p_info.psdata_len, psdata); 1233e170f409SKaricheri, Muralidharan tmp |= (p_info.psdata_len & KNAV_DMA_DESC_PSLEN_MASK) << 123484640e27SKaricheri, Muralidharan KNAV_DMA_DESC_PSLEN_SHIFT; 123584640e27SKaricheri, Muralidharan } 123684640e27SKaricheri, Muralidharan 1237e170f409SKaricheri, Muralidharan tmp |= KNAV_DMA_DESC_HAS_EPIB | 123884640e27SKaricheri, Muralidharan ((netcp->tx_compl_qid & KNAV_DMA_DESC_RETQ_MASK) << 1239e170f409SKaricheri, Muralidharan KNAV_DMA_DESC_RETQ_SHIFT); 124084640e27SKaricheri, Muralidharan 1241e170f409SKaricheri, Muralidharan if (!(tx_pipe->flags & SWITCH_TO_PORT_IN_TAGINFO)) { 1242e170f409SKaricheri, Muralidharan tmp |= ((tx_pipe->switch_to_port & KNAV_DMA_DESC_PSFLAG_MASK) << 1243e170f409SKaricheri, Muralidharan KNAV_DMA_DESC_PSFLAG_SHIFT); 1244e170f409SKaricheri, Muralidharan } 1245e170f409SKaricheri, Muralidharan 1246e170f409SKaricheri, Muralidharan set_words(&tmp, 1, &desc->packet_info); 124706324481SKaricheri, Muralidharan /* warning!!!! We are saving the virtual ptr in the sw_data 124806324481SKaricheri, Muralidharan * field as a 32bit value. Will not work on 64bit machines 124906324481SKaricheri, Muralidharan */ 125006324481SKaricheri, Muralidharan SET_SW_DATA0((u32)skb, desc); 125184640e27SKaricheri, Muralidharan 1252e170f409SKaricheri, Muralidharan if (tx_pipe->flags & SWITCH_TO_PORT_IN_TAGINFO) { 1253e170f409SKaricheri, Muralidharan tmp = tx_pipe->switch_to_port; 125489907779SArnd Bergmann set_words(&tmp, 1, &desc->tag_info); 1255e170f409SKaricheri, Muralidharan } 1256e170f409SKaricheri, Muralidharan 125784640e27SKaricheri, Muralidharan /* submit packet descriptor */ 125884640e27SKaricheri, Muralidharan ret = knav_pool_desc_map(netcp->tx_pool, desc, sizeof(*desc), &dma, 125984640e27SKaricheri, Muralidharan &dma_sz); 126084640e27SKaricheri, Muralidharan if (unlikely(ret)) { 126184640e27SKaricheri, Muralidharan dev_err(netcp->ndev_dev, "%s() failed to map desc\n", __func__); 126284640e27SKaricheri, Muralidharan ret = -ENOMEM; 126384640e27SKaricheri, Muralidharan goto out; 126484640e27SKaricheri, Muralidharan } 126584640e27SKaricheri, Muralidharan skb_tx_timestamp(skb); 126684640e27SKaricheri, Muralidharan knav_queue_push(tx_pipe->dma_queue, dma, dma_sz, 0); 126784640e27SKaricheri, Muralidharan 126884640e27SKaricheri, Muralidharan out: 126984640e27SKaricheri, Muralidharan return ret; 127084640e27SKaricheri, Muralidharan } 127184640e27SKaricheri, Muralidharan 127284640e27SKaricheri, Muralidharan /* Submit the packet */ 127384640e27SKaricheri, Muralidharan static int netcp_ndo_start_xmit(struct sk_buff *skb, struct net_device *ndev) 127484640e27SKaricheri, Muralidharan { 127584640e27SKaricheri, Muralidharan struct netcp_intf *netcp = netdev_priv(ndev); 12766a8162e9SMichael Scherban struct netcp_stats *tx_stats = &netcp->stats; 127784640e27SKaricheri, Muralidharan int subqueue = skb_get_queue_mapping(skb); 127884640e27SKaricheri, Muralidharan struct knav_dma_desc *desc; 127984640e27SKaricheri, Muralidharan int desc_count, ret = 0; 128084640e27SKaricheri, Muralidharan 128184640e27SKaricheri, Muralidharan if (unlikely(skb->len <= 0)) { 128284640e27SKaricheri, Muralidharan dev_kfree_skb(skb); 128384640e27SKaricheri, Muralidharan return NETDEV_TX_OK; 128484640e27SKaricheri, Muralidharan } 128584640e27SKaricheri, Muralidharan 128684640e27SKaricheri, Muralidharan if (unlikely(skb->len < NETCP_MIN_PACKET_SIZE)) { 128784640e27SKaricheri, Muralidharan ret = skb_padto(skb, NETCP_MIN_PACKET_SIZE); 128884640e27SKaricheri, Muralidharan if (ret < 0) { 128984640e27SKaricheri, Muralidharan /* If we get here, the skb has already been dropped */ 129084640e27SKaricheri, Muralidharan dev_warn(netcp->ndev_dev, "padding failed (%d), packet dropped\n", 129184640e27SKaricheri, Muralidharan ret); 12926a8162e9SMichael Scherban tx_stats->tx_dropped++; 129384640e27SKaricheri, Muralidharan return ret; 129484640e27SKaricheri, Muralidharan } 129584640e27SKaricheri, Muralidharan skb->len = NETCP_MIN_PACKET_SIZE; 129684640e27SKaricheri, Muralidharan } 129784640e27SKaricheri, Muralidharan 129884640e27SKaricheri, Muralidharan desc = netcp_tx_map_skb(skb, netcp); 129984640e27SKaricheri, Muralidharan if (unlikely(!desc)) { 130084640e27SKaricheri, Muralidharan netif_stop_subqueue(ndev, subqueue); 130184640e27SKaricheri, Muralidharan ret = -ENOBUFS; 130284640e27SKaricheri, Muralidharan goto drop; 130384640e27SKaricheri, Muralidharan } 130484640e27SKaricheri, Muralidharan 130584640e27SKaricheri, Muralidharan ret = netcp_tx_submit_skb(netcp, skb, desc); 130684640e27SKaricheri, Muralidharan if (ret) 130784640e27SKaricheri, Muralidharan goto drop; 130884640e27SKaricheri, Muralidharan 130984640e27SKaricheri, Muralidharan /* Check Tx pool count & stop subqueue if needed */ 131084640e27SKaricheri, Muralidharan desc_count = knav_pool_count(netcp->tx_pool); 131184640e27SKaricheri, Muralidharan if (desc_count < netcp->tx_pause_threshold) { 131284640e27SKaricheri, Muralidharan dev_dbg(netcp->ndev_dev, "pausing tx, count(%d)\n", desc_count); 131384640e27SKaricheri, Muralidharan netif_stop_subqueue(ndev, subqueue); 131484640e27SKaricheri, Muralidharan } 131584640e27SKaricheri, Muralidharan return NETDEV_TX_OK; 131684640e27SKaricheri, Muralidharan 131784640e27SKaricheri, Muralidharan drop: 13186a8162e9SMichael Scherban tx_stats->tx_dropped++; 131984640e27SKaricheri, Muralidharan if (desc) 132084640e27SKaricheri, Muralidharan netcp_free_tx_desc_chain(netcp, desc, sizeof(*desc)); 132184640e27SKaricheri, Muralidharan dev_kfree_skb(skb); 132284640e27SKaricheri, Muralidharan return ret; 132384640e27SKaricheri, Muralidharan } 132484640e27SKaricheri, Muralidharan 132584640e27SKaricheri, Muralidharan int netcp_txpipe_close(struct netcp_tx_pipe *tx_pipe) 132684640e27SKaricheri, Muralidharan { 132784640e27SKaricheri, Muralidharan if (tx_pipe->dma_channel) { 132884640e27SKaricheri, Muralidharan knav_dma_close_channel(tx_pipe->dma_channel); 132984640e27SKaricheri, Muralidharan tx_pipe->dma_channel = NULL; 133084640e27SKaricheri, Muralidharan } 133184640e27SKaricheri, Muralidharan return 0; 133284640e27SKaricheri, Muralidharan } 133358c11b5fSKaricheri, Muralidharan EXPORT_SYMBOL_GPL(netcp_txpipe_close); 133484640e27SKaricheri, Muralidharan 133584640e27SKaricheri, Muralidharan int netcp_txpipe_open(struct netcp_tx_pipe *tx_pipe) 133684640e27SKaricheri, Muralidharan { 133784640e27SKaricheri, Muralidharan struct device *dev = tx_pipe->netcp_device->device; 133884640e27SKaricheri, Muralidharan struct knav_dma_cfg config; 133984640e27SKaricheri, Muralidharan int ret = 0; 134084640e27SKaricheri, Muralidharan u8 name[16]; 134184640e27SKaricheri, Muralidharan 134284640e27SKaricheri, Muralidharan memset(&config, 0, sizeof(config)); 134384640e27SKaricheri, Muralidharan config.direction = DMA_MEM_TO_DEV; 134484640e27SKaricheri, Muralidharan config.u.tx.filt_einfo = false; 134584640e27SKaricheri, Muralidharan config.u.tx.filt_pswords = false; 134684640e27SKaricheri, Muralidharan config.u.tx.priority = DMA_PRIO_MED_L; 134784640e27SKaricheri, Muralidharan 134884640e27SKaricheri, Muralidharan tx_pipe->dma_channel = knav_dma_open_channel(dev, 134984640e27SKaricheri, Muralidharan tx_pipe->dma_chan_name, &config); 13505b6cb43bSIvan Khoronzhuk if (IS_ERR(tx_pipe->dma_channel)) { 135184640e27SKaricheri, Muralidharan dev_err(dev, "failed opening tx chan(%s)\n", 135284640e27SKaricheri, Muralidharan tx_pipe->dma_chan_name); 13535b6cb43bSIvan Khoronzhuk ret = PTR_ERR(tx_pipe->dma_channel); 135484640e27SKaricheri, Muralidharan goto err; 135584640e27SKaricheri, Muralidharan } 135684640e27SKaricheri, Muralidharan 135784640e27SKaricheri, Muralidharan snprintf(name, sizeof(name), "tx-pipe-%s", dev_name(dev)); 135884640e27SKaricheri, Muralidharan tx_pipe->dma_queue = knav_queue_open(name, tx_pipe->dma_queue_id, 135984640e27SKaricheri, Muralidharan KNAV_QUEUE_SHARED); 136084640e27SKaricheri, Muralidharan if (IS_ERR(tx_pipe->dma_queue)) { 136184640e27SKaricheri, Muralidharan dev_err(dev, "Could not open DMA queue for channel \"%s\": %d\n", 136284640e27SKaricheri, Muralidharan name, ret); 136384640e27SKaricheri, Muralidharan ret = PTR_ERR(tx_pipe->dma_queue); 136484640e27SKaricheri, Muralidharan goto err; 136584640e27SKaricheri, Muralidharan } 136684640e27SKaricheri, Muralidharan 136784640e27SKaricheri, Muralidharan dev_dbg(dev, "opened tx pipe %s\n", name); 136884640e27SKaricheri, Muralidharan return 0; 136984640e27SKaricheri, Muralidharan 137084640e27SKaricheri, Muralidharan err: 137184640e27SKaricheri, Muralidharan if (!IS_ERR_OR_NULL(tx_pipe->dma_channel)) 137284640e27SKaricheri, Muralidharan knav_dma_close_channel(tx_pipe->dma_channel); 137384640e27SKaricheri, Muralidharan tx_pipe->dma_channel = NULL; 137484640e27SKaricheri, Muralidharan return ret; 137584640e27SKaricheri, Muralidharan } 137658c11b5fSKaricheri, Muralidharan EXPORT_SYMBOL_GPL(netcp_txpipe_open); 137784640e27SKaricheri, Muralidharan 137884640e27SKaricheri, Muralidharan int netcp_txpipe_init(struct netcp_tx_pipe *tx_pipe, 137984640e27SKaricheri, Muralidharan struct netcp_device *netcp_device, 138084640e27SKaricheri, Muralidharan const char *dma_chan_name, unsigned int dma_queue_id) 138184640e27SKaricheri, Muralidharan { 138284640e27SKaricheri, Muralidharan memset(tx_pipe, 0, sizeof(*tx_pipe)); 138384640e27SKaricheri, Muralidharan tx_pipe->netcp_device = netcp_device; 138484640e27SKaricheri, Muralidharan tx_pipe->dma_chan_name = dma_chan_name; 138584640e27SKaricheri, Muralidharan tx_pipe->dma_queue_id = dma_queue_id; 138684640e27SKaricheri, Muralidharan return 0; 138784640e27SKaricheri, Muralidharan } 138858c11b5fSKaricheri, Muralidharan EXPORT_SYMBOL_GPL(netcp_txpipe_init); 138984640e27SKaricheri, Muralidharan 139084640e27SKaricheri, Muralidharan static struct netcp_addr *netcp_addr_find(struct netcp_intf *netcp, 139184640e27SKaricheri, Muralidharan const u8 *addr, 139284640e27SKaricheri, Muralidharan enum netcp_addr_type type) 139384640e27SKaricheri, Muralidharan { 139484640e27SKaricheri, Muralidharan struct netcp_addr *naddr; 139584640e27SKaricheri, Muralidharan 139684640e27SKaricheri, Muralidharan list_for_each_entry(naddr, &netcp->addr_list, node) { 139784640e27SKaricheri, Muralidharan if (naddr->type != type) 139884640e27SKaricheri, Muralidharan continue; 139984640e27SKaricheri, Muralidharan if (addr && memcmp(addr, naddr->addr, ETH_ALEN)) 140084640e27SKaricheri, Muralidharan continue; 140184640e27SKaricheri, Muralidharan return naddr; 140284640e27SKaricheri, Muralidharan } 140384640e27SKaricheri, Muralidharan 140484640e27SKaricheri, Muralidharan return NULL; 140584640e27SKaricheri, Muralidharan } 140684640e27SKaricheri, Muralidharan 140784640e27SKaricheri, Muralidharan static struct netcp_addr *netcp_addr_add(struct netcp_intf *netcp, 140884640e27SKaricheri, Muralidharan const u8 *addr, 140984640e27SKaricheri, Muralidharan enum netcp_addr_type type) 141084640e27SKaricheri, Muralidharan { 141184640e27SKaricheri, Muralidharan struct netcp_addr *naddr; 141284640e27SKaricheri, Muralidharan 141384640e27SKaricheri, Muralidharan naddr = devm_kmalloc(netcp->dev, sizeof(*naddr), GFP_ATOMIC); 141484640e27SKaricheri, Muralidharan if (!naddr) 141584640e27SKaricheri, Muralidharan return NULL; 141684640e27SKaricheri, Muralidharan 141784640e27SKaricheri, Muralidharan naddr->type = type; 141884640e27SKaricheri, Muralidharan naddr->flags = 0; 141984640e27SKaricheri, Muralidharan naddr->netcp = netcp; 142084640e27SKaricheri, Muralidharan if (addr) 142184640e27SKaricheri, Muralidharan ether_addr_copy(naddr->addr, addr); 142284640e27SKaricheri, Muralidharan else 1423c7bf7169SJoe Perches eth_zero_addr(naddr->addr); 142484640e27SKaricheri, Muralidharan list_add_tail(&naddr->node, &netcp->addr_list); 142584640e27SKaricheri, Muralidharan 142684640e27SKaricheri, Muralidharan return naddr; 142784640e27SKaricheri, Muralidharan } 142884640e27SKaricheri, Muralidharan 142984640e27SKaricheri, Muralidharan static void netcp_addr_del(struct netcp_intf *netcp, struct netcp_addr *naddr) 143084640e27SKaricheri, Muralidharan { 143184640e27SKaricheri, Muralidharan list_del(&naddr->node); 143284640e27SKaricheri, Muralidharan devm_kfree(netcp->dev, naddr); 143384640e27SKaricheri, Muralidharan } 143484640e27SKaricheri, Muralidharan 143584640e27SKaricheri, Muralidharan static void netcp_addr_clear_mark(struct netcp_intf *netcp) 143684640e27SKaricheri, Muralidharan { 143784640e27SKaricheri, Muralidharan struct netcp_addr *naddr; 143884640e27SKaricheri, Muralidharan 143984640e27SKaricheri, Muralidharan list_for_each_entry(naddr, &netcp->addr_list, node) 144084640e27SKaricheri, Muralidharan naddr->flags = 0; 144184640e27SKaricheri, Muralidharan } 144284640e27SKaricheri, Muralidharan 144384640e27SKaricheri, Muralidharan static void netcp_addr_add_mark(struct netcp_intf *netcp, const u8 *addr, 144484640e27SKaricheri, Muralidharan enum netcp_addr_type type) 144584640e27SKaricheri, Muralidharan { 144684640e27SKaricheri, Muralidharan struct netcp_addr *naddr; 144784640e27SKaricheri, Muralidharan 144884640e27SKaricheri, Muralidharan naddr = netcp_addr_find(netcp, addr, type); 144984640e27SKaricheri, Muralidharan if (naddr) { 145084640e27SKaricheri, Muralidharan naddr->flags |= ADDR_VALID; 145184640e27SKaricheri, Muralidharan return; 145284640e27SKaricheri, Muralidharan } 145384640e27SKaricheri, Muralidharan 145484640e27SKaricheri, Muralidharan naddr = netcp_addr_add(netcp, addr, type); 145584640e27SKaricheri, Muralidharan if (!WARN_ON(!naddr)) 145684640e27SKaricheri, Muralidharan naddr->flags |= ADDR_NEW; 145784640e27SKaricheri, Muralidharan } 145884640e27SKaricheri, Muralidharan 145984640e27SKaricheri, Muralidharan static void netcp_addr_sweep_del(struct netcp_intf *netcp) 146084640e27SKaricheri, Muralidharan { 146184640e27SKaricheri, Muralidharan struct netcp_addr *naddr, *tmp; 146284640e27SKaricheri, Muralidharan struct netcp_intf_modpriv *priv; 146384640e27SKaricheri, Muralidharan struct netcp_module *module; 146484640e27SKaricheri, Muralidharan int error; 146584640e27SKaricheri, Muralidharan 146684640e27SKaricheri, Muralidharan list_for_each_entry_safe(naddr, tmp, &netcp->addr_list, node) { 146784640e27SKaricheri, Muralidharan if (naddr->flags & (ADDR_VALID | ADDR_NEW)) 146884640e27SKaricheri, Muralidharan continue; 146984640e27SKaricheri, Muralidharan dev_dbg(netcp->ndev_dev, "deleting address %pM, type %x\n", 147084640e27SKaricheri, Muralidharan naddr->addr, naddr->type); 147184640e27SKaricheri, Muralidharan for_each_module(netcp, priv) { 147284640e27SKaricheri, Muralidharan module = priv->netcp_module; 147384640e27SKaricheri, Muralidharan if (!module->del_addr) 147484640e27SKaricheri, Muralidharan continue; 147584640e27SKaricheri, Muralidharan error = module->del_addr(priv->module_priv, 147684640e27SKaricheri, Muralidharan naddr); 147784640e27SKaricheri, Muralidharan WARN_ON(error); 147884640e27SKaricheri, Muralidharan } 147984640e27SKaricheri, Muralidharan netcp_addr_del(netcp, naddr); 148084640e27SKaricheri, Muralidharan } 148184640e27SKaricheri, Muralidharan } 148284640e27SKaricheri, Muralidharan 148384640e27SKaricheri, Muralidharan static void netcp_addr_sweep_add(struct netcp_intf *netcp) 148484640e27SKaricheri, Muralidharan { 148584640e27SKaricheri, Muralidharan struct netcp_addr *naddr, *tmp; 148684640e27SKaricheri, Muralidharan struct netcp_intf_modpriv *priv; 148784640e27SKaricheri, Muralidharan struct netcp_module *module; 148884640e27SKaricheri, Muralidharan int error; 148984640e27SKaricheri, Muralidharan 149084640e27SKaricheri, Muralidharan list_for_each_entry_safe(naddr, tmp, &netcp->addr_list, node) { 149184640e27SKaricheri, Muralidharan if (!(naddr->flags & ADDR_NEW)) 149284640e27SKaricheri, Muralidharan continue; 149384640e27SKaricheri, Muralidharan dev_dbg(netcp->ndev_dev, "adding address %pM, type %x\n", 149484640e27SKaricheri, Muralidharan naddr->addr, naddr->type); 14958ceaf361SKaricheri, Muralidharan 149684640e27SKaricheri, Muralidharan for_each_module(netcp, priv) { 149784640e27SKaricheri, Muralidharan module = priv->netcp_module; 149884640e27SKaricheri, Muralidharan if (!module->add_addr) 149984640e27SKaricheri, Muralidharan continue; 150084640e27SKaricheri, Muralidharan error = module->add_addr(priv->module_priv, naddr); 150184640e27SKaricheri, Muralidharan WARN_ON(error); 150284640e27SKaricheri, Muralidharan } 150384640e27SKaricheri, Muralidharan } 150484640e27SKaricheri, Muralidharan } 150584640e27SKaricheri, Muralidharan 15060542a87cSWingMan Kwok static int netcp_set_promiscuous(struct netcp_intf *netcp, bool promisc) 15070542a87cSWingMan Kwok { 15080542a87cSWingMan Kwok struct netcp_intf_modpriv *priv; 15090542a87cSWingMan Kwok struct netcp_module *module; 15100542a87cSWingMan Kwok int error; 15110542a87cSWingMan Kwok 15120542a87cSWingMan Kwok for_each_module(netcp, priv) { 15130542a87cSWingMan Kwok module = priv->netcp_module; 15140542a87cSWingMan Kwok if (!module->set_rx_mode) 15150542a87cSWingMan Kwok continue; 15160542a87cSWingMan Kwok 15170542a87cSWingMan Kwok error = module->set_rx_mode(priv->module_priv, promisc); 15180542a87cSWingMan Kwok if (error) 15190542a87cSWingMan Kwok return error; 15200542a87cSWingMan Kwok } 15210542a87cSWingMan Kwok return 0; 15220542a87cSWingMan Kwok } 15230542a87cSWingMan Kwok 152484640e27SKaricheri, Muralidharan static void netcp_set_rx_mode(struct net_device *ndev) 152584640e27SKaricheri, Muralidharan { 152684640e27SKaricheri, Muralidharan struct netcp_intf *netcp = netdev_priv(ndev); 152784640e27SKaricheri, Muralidharan struct netdev_hw_addr *ndev_addr; 152884640e27SKaricheri, Muralidharan bool promisc; 152984640e27SKaricheri, Muralidharan 153084640e27SKaricheri, Muralidharan promisc = (ndev->flags & IFF_PROMISC || 153184640e27SKaricheri, Muralidharan ndev->flags & IFF_ALLMULTI || 153284640e27SKaricheri, Muralidharan netdev_mc_count(ndev) > NETCP_MAX_MCAST_ADDR); 153384640e27SKaricheri, Muralidharan 15348ceaf361SKaricheri, Muralidharan spin_lock(&netcp->lock); 153584640e27SKaricheri, Muralidharan /* first clear all marks */ 153684640e27SKaricheri, Muralidharan netcp_addr_clear_mark(netcp); 153784640e27SKaricheri, Muralidharan 153884640e27SKaricheri, Muralidharan /* next add new entries, mark existing ones */ 153984640e27SKaricheri, Muralidharan netcp_addr_add_mark(netcp, ndev->broadcast, ADDR_BCAST); 154084640e27SKaricheri, Muralidharan for_each_dev_addr(ndev, ndev_addr) 154184640e27SKaricheri, Muralidharan netcp_addr_add_mark(netcp, ndev_addr->addr, ADDR_DEV); 154284640e27SKaricheri, Muralidharan netdev_for_each_uc_addr(ndev_addr, ndev) 154384640e27SKaricheri, Muralidharan netcp_addr_add_mark(netcp, ndev_addr->addr, ADDR_UCAST); 154484640e27SKaricheri, Muralidharan netdev_for_each_mc_addr(ndev_addr, ndev) 154584640e27SKaricheri, Muralidharan netcp_addr_add_mark(netcp, ndev_addr->addr, ADDR_MCAST); 154684640e27SKaricheri, Muralidharan 154784640e27SKaricheri, Muralidharan if (promisc) 154884640e27SKaricheri, Muralidharan netcp_addr_add_mark(netcp, NULL, ADDR_ANY); 154984640e27SKaricheri, Muralidharan 155084640e27SKaricheri, Muralidharan /* finally sweep and callout into modules */ 155184640e27SKaricheri, Muralidharan netcp_addr_sweep_del(netcp); 155284640e27SKaricheri, Muralidharan netcp_addr_sweep_add(netcp); 15530542a87cSWingMan Kwok netcp_set_promiscuous(netcp, promisc); 15548ceaf361SKaricheri, Muralidharan spin_unlock(&netcp->lock); 155584640e27SKaricheri, Muralidharan } 155684640e27SKaricheri, Muralidharan 155784640e27SKaricheri, Muralidharan static void netcp_free_navigator_resources(struct netcp_intf *netcp) 155884640e27SKaricheri, Muralidharan { 155984640e27SKaricheri, Muralidharan int i; 156084640e27SKaricheri, Muralidharan 156184640e27SKaricheri, Muralidharan if (netcp->rx_channel) { 156284640e27SKaricheri, Muralidharan knav_dma_close_channel(netcp->rx_channel); 156384640e27SKaricheri, Muralidharan netcp->rx_channel = NULL; 156484640e27SKaricheri, Muralidharan } 156584640e27SKaricheri, Muralidharan 156684640e27SKaricheri, Muralidharan if (!IS_ERR_OR_NULL(netcp->rx_pool)) 156784640e27SKaricheri, Muralidharan netcp_rxpool_free(netcp); 156884640e27SKaricheri, Muralidharan 156984640e27SKaricheri, Muralidharan if (!IS_ERR_OR_NULL(netcp->rx_queue)) { 157084640e27SKaricheri, Muralidharan knav_queue_close(netcp->rx_queue); 157184640e27SKaricheri, Muralidharan netcp->rx_queue = NULL; 157284640e27SKaricheri, Muralidharan } 157384640e27SKaricheri, Muralidharan 157484640e27SKaricheri, Muralidharan for (i = 0; i < KNAV_DMA_FDQ_PER_CHAN && 157584640e27SKaricheri, Muralidharan !IS_ERR_OR_NULL(netcp->rx_fdq[i]) ; ++i) { 157684640e27SKaricheri, Muralidharan knav_queue_close(netcp->rx_fdq[i]); 157784640e27SKaricheri, Muralidharan netcp->rx_fdq[i] = NULL; 157884640e27SKaricheri, Muralidharan } 157984640e27SKaricheri, Muralidharan 158084640e27SKaricheri, Muralidharan if (!IS_ERR_OR_NULL(netcp->tx_compl_q)) { 158184640e27SKaricheri, Muralidharan knav_queue_close(netcp->tx_compl_q); 158284640e27SKaricheri, Muralidharan netcp->tx_compl_q = NULL; 158384640e27SKaricheri, Muralidharan } 158484640e27SKaricheri, Muralidharan 158584640e27SKaricheri, Muralidharan if (!IS_ERR_OR_NULL(netcp->tx_pool)) { 158684640e27SKaricheri, Muralidharan knav_pool_destroy(netcp->tx_pool); 158784640e27SKaricheri, Muralidharan netcp->tx_pool = NULL; 158884640e27SKaricheri, Muralidharan } 158984640e27SKaricheri, Muralidharan } 159084640e27SKaricheri, Muralidharan 159184640e27SKaricheri, Muralidharan static int netcp_setup_navigator_resources(struct net_device *ndev) 159284640e27SKaricheri, Muralidharan { 159384640e27SKaricheri, Muralidharan struct netcp_intf *netcp = netdev_priv(ndev); 159484640e27SKaricheri, Muralidharan struct knav_queue_notify_config notify_cfg; 159584640e27SKaricheri, Muralidharan struct knav_dma_cfg config; 159684640e27SKaricheri, Muralidharan u32 last_fdq = 0; 159784640e27SKaricheri, Muralidharan u8 name[16]; 159884640e27SKaricheri, Muralidharan int ret; 159984640e27SKaricheri, Muralidharan int i; 160084640e27SKaricheri, Muralidharan 160184640e27SKaricheri, Muralidharan /* Create Rx/Tx descriptor pools */ 160284640e27SKaricheri, Muralidharan snprintf(name, sizeof(name), "rx-pool-%s", ndev->name); 160384640e27SKaricheri, Muralidharan netcp->rx_pool = knav_pool_create(name, netcp->rx_pool_size, 160484640e27SKaricheri, Muralidharan netcp->rx_pool_region_id); 160584640e27SKaricheri, Muralidharan if (IS_ERR_OR_NULL(netcp->rx_pool)) { 160684640e27SKaricheri, Muralidharan dev_err(netcp->ndev_dev, "Couldn't create rx pool\n"); 160784640e27SKaricheri, Muralidharan ret = PTR_ERR(netcp->rx_pool); 160884640e27SKaricheri, Muralidharan goto fail; 160984640e27SKaricheri, Muralidharan } 161084640e27SKaricheri, Muralidharan 161184640e27SKaricheri, Muralidharan snprintf(name, sizeof(name), "tx-pool-%s", ndev->name); 161284640e27SKaricheri, Muralidharan netcp->tx_pool = knav_pool_create(name, netcp->tx_pool_size, 161384640e27SKaricheri, Muralidharan netcp->tx_pool_region_id); 161484640e27SKaricheri, Muralidharan if (IS_ERR_OR_NULL(netcp->tx_pool)) { 161584640e27SKaricheri, Muralidharan dev_err(netcp->ndev_dev, "Couldn't create tx pool\n"); 161684640e27SKaricheri, Muralidharan ret = PTR_ERR(netcp->tx_pool); 161784640e27SKaricheri, Muralidharan goto fail; 161884640e27SKaricheri, Muralidharan } 161984640e27SKaricheri, Muralidharan 162084640e27SKaricheri, Muralidharan /* open Tx completion queue */ 162184640e27SKaricheri, Muralidharan snprintf(name, sizeof(name), "tx-compl-%s", ndev->name); 162284640e27SKaricheri, Muralidharan netcp->tx_compl_q = knav_queue_open(name, netcp->tx_compl_qid, 0); 1623d01eb808SJulia Lawall if (IS_ERR(netcp->tx_compl_q)) { 162484640e27SKaricheri, Muralidharan ret = PTR_ERR(netcp->tx_compl_q); 162584640e27SKaricheri, Muralidharan goto fail; 162684640e27SKaricheri, Muralidharan } 162784640e27SKaricheri, Muralidharan netcp->tx_compl_qid = knav_queue_get_id(netcp->tx_compl_q); 162884640e27SKaricheri, Muralidharan 162984640e27SKaricheri, Muralidharan /* Set notification for Tx completion */ 163084640e27SKaricheri, Muralidharan notify_cfg.fn = netcp_tx_notify; 163184640e27SKaricheri, Muralidharan notify_cfg.fn_arg = netcp; 163284640e27SKaricheri, Muralidharan ret = knav_queue_device_control(netcp->tx_compl_q, 163384640e27SKaricheri, Muralidharan KNAV_QUEUE_SET_NOTIFIER, 163484640e27SKaricheri, Muralidharan (unsigned long)¬ify_cfg); 163584640e27SKaricheri, Muralidharan if (ret) 163684640e27SKaricheri, Muralidharan goto fail; 163784640e27SKaricheri, Muralidharan 163884640e27SKaricheri, Muralidharan knav_queue_disable_notify(netcp->tx_compl_q); 163984640e27SKaricheri, Muralidharan 164084640e27SKaricheri, Muralidharan /* open Rx completion queue */ 164184640e27SKaricheri, Muralidharan snprintf(name, sizeof(name), "rx-compl-%s", ndev->name); 164284640e27SKaricheri, Muralidharan netcp->rx_queue = knav_queue_open(name, netcp->rx_queue_id, 0); 1643d01eb808SJulia Lawall if (IS_ERR(netcp->rx_queue)) { 164484640e27SKaricheri, Muralidharan ret = PTR_ERR(netcp->rx_queue); 164584640e27SKaricheri, Muralidharan goto fail; 164684640e27SKaricheri, Muralidharan } 164784640e27SKaricheri, Muralidharan netcp->rx_queue_id = knav_queue_get_id(netcp->rx_queue); 164884640e27SKaricheri, Muralidharan 164984640e27SKaricheri, Muralidharan /* Set notification for Rx completion */ 165084640e27SKaricheri, Muralidharan notify_cfg.fn = netcp_rx_notify; 165184640e27SKaricheri, Muralidharan notify_cfg.fn_arg = netcp; 165284640e27SKaricheri, Muralidharan ret = knav_queue_device_control(netcp->rx_queue, 165384640e27SKaricheri, Muralidharan KNAV_QUEUE_SET_NOTIFIER, 165484640e27SKaricheri, Muralidharan (unsigned long)¬ify_cfg); 165584640e27SKaricheri, Muralidharan if (ret) 165684640e27SKaricheri, Muralidharan goto fail; 165784640e27SKaricheri, Muralidharan 165884640e27SKaricheri, Muralidharan knav_queue_disable_notify(netcp->rx_queue); 165984640e27SKaricheri, Muralidharan 166084640e27SKaricheri, Muralidharan /* open Rx FDQs */ 1661866b8b18SWingMan Kwok for (i = 0; i < KNAV_DMA_FDQ_PER_CHAN && netcp->rx_queue_depths[i]; 1662866b8b18SWingMan Kwok ++i) { 166384640e27SKaricheri, Muralidharan snprintf(name, sizeof(name), "rx-fdq-%s-%d", ndev->name, i); 166484640e27SKaricheri, Muralidharan netcp->rx_fdq[i] = knav_queue_open(name, KNAV_QUEUE_GP, 0); 1665d01eb808SJulia Lawall if (IS_ERR(netcp->rx_fdq[i])) { 166684640e27SKaricheri, Muralidharan ret = PTR_ERR(netcp->rx_fdq[i]); 166784640e27SKaricheri, Muralidharan goto fail; 166884640e27SKaricheri, Muralidharan } 166984640e27SKaricheri, Muralidharan } 167084640e27SKaricheri, Muralidharan 167184640e27SKaricheri, Muralidharan memset(&config, 0, sizeof(config)); 167284640e27SKaricheri, Muralidharan config.direction = DMA_DEV_TO_MEM; 167384640e27SKaricheri, Muralidharan config.u.rx.einfo_present = true; 167484640e27SKaricheri, Muralidharan config.u.rx.psinfo_present = true; 167584640e27SKaricheri, Muralidharan config.u.rx.err_mode = DMA_DROP; 167684640e27SKaricheri, Muralidharan config.u.rx.desc_type = DMA_DESC_HOST; 167784640e27SKaricheri, Muralidharan config.u.rx.psinfo_at_sop = false; 167884640e27SKaricheri, Muralidharan config.u.rx.sop_offset = NETCP_SOP_OFFSET; 167984640e27SKaricheri, Muralidharan config.u.rx.dst_q = netcp->rx_queue_id; 168084640e27SKaricheri, Muralidharan config.u.rx.thresh = DMA_THRESH_NONE; 168184640e27SKaricheri, Muralidharan 168284640e27SKaricheri, Muralidharan for (i = 0; i < KNAV_DMA_FDQ_PER_CHAN; ++i) { 168384640e27SKaricheri, Muralidharan if (netcp->rx_fdq[i]) 168484640e27SKaricheri, Muralidharan last_fdq = knav_queue_get_id(netcp->rx_fdq[i]); 168584640e27SKaricheri, Muralidharan config.u.rx.fdq[i] = last_fdq; 168684640e27SKaricheri, Muralidharan } 168784640e27SKaricheri, Muralidharan 168884640e27SKaricheri, Muralidharan netcp->rx_channel = knav_dma_open_channel(netcp->netcp_device->device, 168984640e27SKaricheri, Muralidharan netcp->dma_chan_name, &config); 16905b6cb43bSIvan Khoronzhuk if (IS_ERR(netcp->rx_channel)) { 169184640e27SKaricheri, Muralidharan dev_err(netcp->ndev_dev, "failed opening rx chan(%s\n", 169284640e27SKaricheri, Muralidharan netcp->dma_chan_name); 16935b6cb43bSIvan Khoronzhuk ret = PTR_ERR(netcp->rx_channel); 169484640e27SKaricheri, Muralidharan goto fail; 169584640e27SKaricheri, Muralidharan } 169684640e27SKaricheri, Muralidharan 169784640e27SKaricheri, Muralidharan dev_dbg(netcp->ndev_dev, "opened RX channel: %p\n", netcp->rx_channel); 169884640e27SKaricheri, Muralidharan return 0; 169984640e27SKaricheri, Muralidharan 170084640e27SKaricheri, Muralidharan fail: 170184640e27SKaricheri, Muralidharan netcp_free_navigator_resources(netcp); 170284640e27SKaricheri, Muralidharan return ret; 170384640e27SKaricheri, Muralidharan } 170484640e27SKaricheri, Muralidharan 170584640e27SKaricheri, Muralidharan /* Open the device */ 170684640e27SKaricheri, Muralidharan static int netcp_ndo_open(struct net_device *ndev) 170784640e27SKaricheri, Muralidharan { 170884640e27SKaricheri, Muralidharan struct netcp_intf *netcp = netdev_priv(ndev); 170984640e27SKaricheri, Muralidharan struct netcp_intf_modpriv *intf_modpriv; 171084640e27SKaricheri, Muralidharan struct netcp_module *module; 171184640e27SKaricheri, Muralidharan int ret; 171284640e27SKaricheri, Muralidharan 171384640e27SKaricheri, Muralidharan netif_carrier_off(ndev); 171484640e27SKaricheri, Muralidharan ret = netcp_setup_navigator_resources(ndev); 171584640e27SKaricheri, Muralidharan if (ret) { 171684640e27SKaricheri, Muralidharan dev_err(netcp->ndev_dev, "Failed to setup navigator resources\n"); 171784640e27SKaricheri, Muralidharan goto fail; 171884640e27SKaricheri, Muralidharan } 171984640e27SKaricheri, Muralidharan 172084640e27SKaricheri, Muralidharan for_each_module(netcp, intf_modpriv) { 172184640e27SKaricheri, Muralidharan module = intf_modpriv->netcp_module; 172284640e27SKaricheri, Muralidharan if (module->open) { 172384640e27SKaricheri, Muralidharan ret = module->open(intf_modpriv->module_priv, ndev); 172484640e27SKaricheri, Muralidharan if (ret != 0) { 172584640e27SKaricheri, Muralidharan dev_err(netcp->ndev_dev, "module open failed\n"); 172684640e27SKaricheri, Muralidharan goto fail_open; 172784640e27SKaricheri, Muralidharan } 172884640e27SKaricheri, Muralidharan } 172984640e27SKaricheri, Muralidharan } 173084640e27SKaricheri, Muralidharan 173184640e27SKaricheri, Muralidharan napi_enable(&netcp->rx_napi); 173284640e27SKaricheri, Muralidharan napi_enable(&netcp->tx_napi); 173384640e27SKaricheri, Muralidharan knav_queue_enable_notify(netcp->tx_compl_q); 173484640e27SKaricheri, Muralidharan knav_queue_enable_notify(netcp->rx_queue); 1735194ac06eSKaricheri, Muralidharan netcp_rxpool_refill(netcp); 173684640e27SKaricheri, Muralidharan netif_tx_wake_all_queues(ndev); 173784640e27SKaricheri, Muralidharan dev_dbg(netcp->ndev_dev, "netcp device %s opened\n", ndev->name); 173884640e27SKaricheri, Muralidharan return 0; 173984640e27SKaricheri, Muralidharan 174084640e27SKaricheri, Muralidharan fail_open: 174184640e27SKaricheri, Muralidharan for_each_module(netcp, intf_modpriv) { 174284640e27SKaricheri, Muralidharan module = intf_modpriv->netcp_module; 174384640e27SKaricheri, Muralidharan if (module->close) 174484640e27SKaricheri, Muralidharan module->close(intf_modpriv->module_priv, ndev); 174584640e27SKaricheri, Muralidharan } 174684640e27SKaricheri, Muralidharan 174784640e27SKaricheri, Muralidharan fail: 174884640e27SKaricheri, Muralidharan netcp_free_navigator_resources(netcp); 174984640e27SKaricheri, Muralidharan return ret; 175084640e27SKaricheri, Muralidharan } 175184640e27SKaricheri, Muralidharan 175284640e27SKaricheri, Muralidharan /* Close the device */ 175384640e27SKaricheri, Muralidharan static int netcp_ndo_stop(struct net_device *ndev) 175484640e27SKaricheri, Muralidharan { 175584640e27SKaricheri, Muralidharan struct netcp_intf *netcp = netdev_priv(ndev); 175684640e27SKaricheri, Muralidharan struct netcp_intf_modpriv *intf_modpriv; 175784640e27SKaricheri, Muralidharan struct netcp_module *module; 175884640e27SKaricheri, Muralidharan int err = 0; 175984640e27SKaricheri, Muralidharan 176084640e27SKaricheri, Muralidharan netif_tx_stop_all_queues(ndev); 176184640e27SKaricheri, Muralidharan netif_carrier_off(ndev); 176284640e27SKaricheri, Muralidharan netcp_addr_clear_mark(netcp); 176384640e27SKaricheri, Muralidharan netcp_addr_sweep_del(netcp); 176484640e27SKaricheri, Muralidharan knav_queue_disable_notify(netcp->rx_queue); 176584640e27SKaricheri, Muralidharan knav_queue_disable_notify(netcp->tx_compl_q); 176684640e27SKaricheri, Muralidharan napi_disable(&netcp->rx_napi); 176784640e27SKaricheri, Muralidharan napi_disable(&netcp->tx_napi); 176884640e27SKaricheri, Muralidharan 176984640e27SKaricheri, Muralidharan for_each_module(netcp, intf_modpriv) { 177084640e27SKaricheri, Muralidharan module = intf_modpriv->netcp_module; 177184640e27SKaricheri, Muralidharan if (module->close) { 177284640e27SKaricheri, Muralidharan err = module->close(intf_modpriv->module_priv, ndev); 177384640e27SKaricheri, Muralidharan if (err != 0) 177484640e27SKaricheri, Muralidharan dev_err(netcp->ndev_dev, "Close failed\n"); 177584640e27SKaricheri, Muralidharan } 177684640e27SKaricheri, Muralidharan } 177784640e27SKaricheri, Muralidharan 177884640e27SKaricheri, Muralidharan /* Recycle Rx descriptors from completion queue */ 177984640e27SKaricheri, Muralidharan netcp_empty_rx_queue(netcp); 178084640e27SKaricheri, Muralidharan 178184640e27SKaricheri, Muralidharan /* Recycle Tx descriptors from completion queue */ 178284640e27SKaricheri, Muralidharan netcp_process_tx_compl_packets(netcp, netcp->tx_pool_size); 178384640e27SKaricheri, Muralidharan 178484640e27SKaricheri, Muralidharan if (knav_pool_count(netcp->tx_pool) != netcp->tx_pool_size) 178584640e27SKaricheri, Muralidharan dev_err(netcp->ndev_dev, "Lost (%d) Tx descs\n", 178684640e27SKaricheri, Muralidharan netcp->tx_pool_size - knav_pool_count(netcp->tx_pool)); 178784640e27SKaricheri, Muralidharan 178884640e27SKaricheri, Muralidharan netcp_free_navigator_resources(netcp); 178984640e27SKaricheri, Muralidharan dev_dbg(netcp->ndev_dev, "netcp device %s stopped\n", ndev->name); 179084640e27SKaricheri, Muralidharan return 0; 179184640e27SKaricheri, Muralidharan } 179284640e27SKaricheri, Muralidharan 179384640e27SKaricheri, Muralidharan static int netcp_ndo_ioctl(struct net_device *ndev, 179484640e27SKaricheri, Muralidharan struct ifreq *req, int cmd) 179584640e27SKaricheri, Muralidharan { 179684640e27SKaricheri, Muralidharan struct netcp_intf *netcp = netdev_priv(ndev); 179784640e27SKaricheri, Muralidharan struct netcp_intf_modpriv *intf_modpriv; 179884640e27SKaricheri, Muralidharan struct netcp_module *module; 179984640e27SKaricheri, Muralidharan int ret = -1, err = -EOPNOTSUPP; 180084640e27SKaricheri, Muralidharan 180184640e27SKaricheri, Muralidharan if (!netif_running(ndev)) 180284640e27SKaricheri, Muralidharan return -EINVAL; 180384640e27SKaricheri, Muralidharan 180484640e27SKaricheri, Muralidharan for_each_module(netcp, intf_modpriv) { 180584640e27SKaricheri, Muralidharan module = intf_modpriv->netcp_module; 180684640e27SKaricheri, Muralidharan if (!module->ioctl) 180784640e27SKaricheri, Muralidharan continue; 180884640e27SKaricheri, Muralidharan 180984640e27SKaricheri, Muralidharan err = module->ioctl(intf_modpriv->module_priv, req, cmd); 181084640e27SKaricheri, Muralidharan if ((err < 0) && (err != -EOPNOTSUPP)) { 181184640e27SKaricheri, Muralidharan ret = err; 181284640e27SKaricheri, Muralidharan goto out; 181384640e27SKaricheri, Muralidharan } 181484640e27SKaricheri, Muralidharan if (err == 0) 181584640e27SKaricheri, Muralidharan ret = err; 181684640e27SKaricheri, Muralidharan } 181784640e27SKaricheri, Muralidharan 181884640e27SKaricheri, Muralidharan out: 181984640e27SKaricheri, Muralidharan return (ret == 0) ? 0 : err; 182084640e27SKaricheri, Muralidharan } 182184640e27SKaricheri, Muralidharan 182284640e27SKaricheri, Muralidharan static void netcp_ndo_tx_timeout(struct net_device *ndev) 182384640e27SKaricheri, Muralidharan { 182484640e27SKaricheri, Muralidharan struct netcp_intf *netcp = netdev_priv(ndev); 182584640e27SKaricheri, Muralidharan unsigned int descs = knav_pool_count(netcp->tx_pool); 182684640e27SKaricheri, Muralidharan 182784640e27SKaricheri, Muralidharan dev_err(netcp->ndev_dev, "transmit timed out tx descs(%d)\n", descs); 182884640e27SKaricheri, Muralidharan netcp_process_tx_compl_packets(netcp, netcp->tx_pool_size); 1829860e9538SFlorian Westphal netif_trans_update(ndev); 183084640e27SKaricheri, Muralidharan netif_tx_wake_all_queues(ndev); 183184640e27SKaricheri, Muralidharan } 183284640e27SKaricheri, Muralidharan 183384640e27SKaricheri, Muralidharan static int netcp_rx_add_vid(struct net_device *ndev, __be16 proto, u16 vid) 183484640e27SKaricheri, Muralidharan { 183584640e27SKaricheri, Muralidharan struct netcp_intf *netcp = netdev_priv(ndev); 183684640e27SKaricheri, Muralidharan struct netcp_intf_modpriv *intf_modpriv; 183784640e27SKaricheri, Muralidharan struct netcp_module *module; 18388ceaf361SKaricheri, Muralidharan unsigned long flags; 183984640e27SKaricheri, Muralidharan int err = 0; 184084640e27SKaricheri, Muralidharan 184184640e27SKaricheri, Muralidharan dev_dbg(netcp->ndev_dev, "adding rx vlan id: %d\n", vid); 184284640e27SKaricheri, Muralidharan 18438ceaf361SKaricheri, Muralidharan spin_lock_irqsave(&netcp->lock, flags); 184484640e27SKaricheri, Muralidharan for_each_module(netcp, intf_modpriv) { 184584640e27SKaricheri, Muralidharan module = intf_modpriv->netcp_module; 184684640e27SKaricheri, Muralidharan if ((module->add_vid) && (vid != 0)) { 184784640e27SKaricheri, Muralidharan err = module->add_vid(intf_modpriv->module_priv, vid); 184884640e27SKaricheri, Muralidharan if (err != 0) { 184984640e27SKaricheri, Muralidharan dev_err(netcp->ndev_dev, "Could not add vlan id = %d\n", 185084640e27SKaricheri, Muralidharan vid); 185184640e27SKaricheri, Muralidharan break; 185284640e27SKaricheri, Muralidharan } 185384640e27SKaricheri, Muralidharan } 185484640e27SKaricheri, Muralidharan } 18558ceaf361SKaricheri, Muralidharan spin_unlock_irqrestore(&netcp->lock, flags); 18568ceaf361SKaricheri, Muralidharan 185784640e27SKaricheri, Muralidharan return err; 185884640e27SKaricheri, Muralidharan } 185984640e27SKaricheri, Muralidharan 186084640e27SKaricheri, Muralidharan static int netcp_rx_kill_vid(struct net_device *ndev, __be16 proto, u16 vid) 186184640e27SKaricheri, Muralidharan { 186284640e27SKaricheri, Muralidharan struct netcp_intf *netcp = netdev_priv(ndev); 186384640e27SKaricheri, Muralidharan struct netcp_intf_modpriv *intf_modpriv; 186484640e27SKaricheri, Muralidharan struct netcp_module *module; 18658ceaf361SKaricheri, Muralidharan unsigned long flags; 186684640e27SKaricheri, Muralidharan int err = 0; 186784640e27SKaricheri, Muralidharan 186884640e27SKaricheri, Muralidharan dev_dbg(netcp->ndev_dev, "removing rx vlan id: %d\n", vid); 186984640e27SKaricheri, Muralidharan 18708ceaf361SKaricheri, Muralidharan spin_lock_irqsave(&netcp->lock, flags); 187184640e27SKaricheri, Muralidharan for_each_module(netcp, intf_modpriv) { 187284640e27SKaricheri, Muralidharan module = intf_modpriv->netcp_module; 187384640e27SKaricheri, Muralidharan if (module->del_vid) { 187484640e27SKaricheri, Muralidharan err = module->del_vid(intf_modpriv->module_priv, vid); 187584640e27SKaricheri, Muralidharan if (err != 0) { 187684640e27SKaricheri, Muralidharan dev_err(netcp->ndev_dev, "Could not delete vlan id = %d\n", 187784640e27SKaricheri, Muralidharan vid); 187884640e27SKaricheri, Muralidharan break; 187984640e27SKaricheri, Muralidharan } 188084640e27SKaricheri, Muralidharan } 188184640e27SKaricheri, Muralidharan } 18828ceaf361SKaricheri, Muralidharan spin_unlock_irqrestore(&netcp->lock, flags); 188384640e27SKaricheri, Muralidharan return err; 188484640e27SKaricheri, Muralidharan } 188584640e27SKaricheri, Muralidharan 18862572ac53SJiri Pirko static int netcp_setup_tc(struct net_device *dev, enum tc_setup_type type, 1887de4784caSJiri Pirko void *type_data) 188884640e27SKaricheri, Muralidharan { 1889de4784caSJiri Pirko struct tc_mqprio_qopt *mqprio = type_data; 189056f36acdSAmritha Nambiar u8 num_tc; 189184640e27SKaricheri, Muralidharan int i; 189284640e27SKaricheri, Muralidharan 189384640e27SKaricheri, Muralidharan /* setup tc must be called under rtnl lock */ 189484640e27SKaricheri, Muralidharan ASSERT_RTNL(); 189584640e27SKaricheri, Muralidharan 1896575ed7d3SNogah Frankel if (type != TC_SETUP_QDISC_MQPRIO) 189738cf0426SJiri Pirko return -EOPNOTSUPP; 1898e4c6734eSJohn Fastabend 1899de4784caSJiri Pirko mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS; 1900de4784caSJiri Pirko num_tc = mqprio->num_tc; 190156f36acdSAmritha Nambiar 190284640e27SKaricheri, Muralidharan /* Sanity-check the number of traffic classes requested */ 190384640e27SKaricheri, Muralidharan if ((dev->real_num_tx_queues <= 1) || 190456f36acdSAmritha Nambiar (dev->real_num_tx_queues < num_tc)) 190584640e27SKaricheri, Muralidharan return -EINVAL; 190684640e27SKaricheri, Muralidharan 190784640e27SKaricheri, Muralidharan /* Configure traffic class to queue mappings */ 190856f36acdSAmritha Nambiar if (num_tc) { 190956f36acdSAmritha Nambiar netdev_set_num_tc(dev, num_tc); 191056f36acdSAmritha Nambiar for (i = 0; i < num_tc; i++) 191184640e27SKaricheri, Muralidharan netdev_set_tc_queue(dev, i, 1, i); 191284640e27SKaricheri, Muralidharan } else { 191384640e27SKaricheri, Muralidharan netdev_reset_tc(dev); 191484640e27SKaricheri, Muralidharan } 191584640e27SKaricheri, Muralidharan 191684640e27SKaricheri, Muralidharan return 0; 191784640e27SKaricheri, Muralidharan } 191884640e27SKaricheri, Muralidharan 1919380043b9SKeerthy static void 19206a8162e9SMichael Scherban netcp_get_stats(struct net_device *ndev, struct rtnl_link_stats64 *stats) 19216a8162e9SMichael Scherban { 19226a8162e9SMichael Scherban struct netcp_intf *netcp = netdev_priv(ndev); 19236a8162e9SMichael Scherban struct netcp_stats *p = &netcp->stats; 19246a8162e9SMichael Scherban u64 rxpackets, rxbytes, txpackets, txbytes; 19256a8162e9SMichael Scherban unsigned int start; 19266a8162e9SMichael Scherban 19276a8162e9SMichael Scherban do { 19286a8162e9SMichael Scherban start = u64_stats_fetch_begin_irq(&p->syncp_rx); 19296a8162e9SMichael Scherban rxpackets = p->rx_packets; 19306a8162e9SMichael Scherban rxbytes = p->rx_bytes; 19316a8162e9SMichael Scherban } while (u64_stats_fetch_retry_irq(&p->syncp_rx, start)); 19326a8162e9SMichael Scherban 19336a8162e9SMichael Scherban do { 19346a8162e9SMichael Scherban start = u64_stats_fetch_begin_irq(&p->syncp_tx); 19356a8162e9SMichael Scherban txpackets = p->tx_packets; 19366a8162e9SMichael Scherban txbytes = p->tx_bytes; 19376a8162e9SMichael Scherban } while (u64_stats_fetch_retry_irq(&p->syncp_tx, start)); 19386a8162e9SMichael Scherban 19396a8162e9SMichael Scherban stats->rx_packets = rxpackets; 19406a8162e9SMichael Scherban stats->rx_bytes = rxbytes; 19416a8162e9SMichael Scherban stats->tx_packets = txpackets; 19426a8162e9SMichael Scherban stats->tx_bytes = txbytes; 19436a8162e9SMichael Scherban 19446a8162e9SMichael Scherban /* The following are stored as 32 bit */ 19456a8162e9SMichael Scherban stats->rx_errors = p->rx_errors; 19466a8162e9SMichael Scherban stats->rx_dropped = p->rx_dropped; 19476a8162e9SMichael Scherban stats->tx_dropped = p->tx_dropped; 19486a8162e9SMichael Scherban } 19496a8162e9SMichael Scherban 195084640e27SKaricheri, Muralidharan static const struct net_device_ops netcp_netdev_ops = { 195184640e27SKaricheri, Muralidharan .ndo_open = netcp_ndo_open, 195284640e27SKaricheri, Muralidharan .ndo_stop = netcp_ndo_stop, 195384640e27SKaricheri, Muralidharan .ndo_start_xmit = netcp_ndo_start_xmit, 195484640e27SKaricheri, Muralidharan .ndo_set_rx_mode = netcp_set_rx_mode, 195584640e27SKaricheri, Muralidharan .ndo_do_ioctl = netcp_ndo_ioctl, 19566a8162e9SMichael Scherban .ndo_get_stats64 = netcp_get_stats, 195784640e27SKaricheri, Muralidharan .ndo_set_mac_address = eth_mac_addr, 195884640e27SKaricheri, Muralidharan .ndo_validate_addr = eth_validate_addr, 195984640e27SKaricheri, Muralidharan .ndo_vlan_rx_add_vid = netcp_rx_add_vid, 196084640e27SKaricheri, Muralidharan .ndo_vlan_rx_kill_vid = netcp_rx_kill_vid, 196184640e27SKaricheri, Muralidharan .ndo_tx_timeout = netcp_ndo_tx_timeout, 1962a4ea8a3dSAlexander Duyck .ndo_select_queue = dev_pick_tx_zero, 196384640e27SKaricheri, Muralidharan .ndo_setup_tc = netcp_setup_tc, 196484640e27SKaricheri, Muralidharan }; 196584640e27SKaricheri, Muralidharan 196684640e27SKaricheri, Muralidharan static int netcp_create_interface(struct netcp_device *netcp_device, 196784640e27SKaricheri, Muralidharan struct device_node *node_interface) 196884640e27SKaricheri, Muralidharan { 196984640e27SKaricheri, Muralidharan struct device *dev = netcp_device->device; 197084640e27SKaricheri, Muralidharan struct device_node *node = dev->of_node; 197184640e27SKaricheri, Muralidharan struct netcp_intf *netcp; 197284640e27SKaricheri, Muralidharan struct net_device *ndev; 197384640e27SKaricheri, Muralidharan resource_size_t size; 197484640e27SKaricheri, Muralidharan struct resource res; 197584640e27SKaricheri, Muralidharan void __iomem *efuse = NULL; 197684640e27SKaricheri, Muralidharan u32 efuse_mac = 0; 197784640e27SKaricheri, Muralidharan const void *mac_addr; 197884640e27SKaricheri, Muralidharan u8 efuse_mac_addr[6]; 197984640e27SKaricheri, Muralidharan u32 temp[2]; 198084640e27SKaricheri, Muralidharan int ret = 0; 198184640e27SKaricheri, Muralidharan 198284640e27SKaricheri, Muralidharan ndev = alloc_etherdev_mqs(sizeof(*netcp), 1, 1); 198384640e27SKaricheri, Muralidharan if (!ndev) { 198484640e27SKaricheri, Muralidharan dev_err(dev, "Error allocating netdev\n"); 198584640e27SKaricheri, Muralidharan return -ENOMEM; 198684640e27SKaricheri, Muralidharan } 198784640e27SKaricheri, Muralidharan 198884640e27SKaricheri, Muralidharan ndev->features |= NETIF_F_SG; 198984640e27SKaricheri, Muralidharan ndev->features |= NETIF_F_HW_VLAN_CTAG_FILTER; 199084640e27SKaricheri, Muralidharan ndev->hw_features = ndev->features; 199184640e27SKaricheri, Muralidharan ndev->vlan_features |= NETIF_F_SG; 199284640e27SKaricheri, Muralidharan 199344770e11SJarod Wilson /* MTU range: 68 - 9486 */ 199444770e11SJarod Wilson ndev->min_mtu = ETH_MIN_MTU; 199544770e11SJarod Wilson ndev->max_mtu = NETCP_MAX_FRAME_SIZE - (ETH_HLEN + ETH_FCS_LEN); 199644770e11SJarod Wilson 199784640e27SKaricheri, Muralidharan netcp = netdev_priv(ndev); 199884640e27SKaricheri, Muralidharan spin_lock_init(&netcp->lock); 199984640e27SKaricheri, Muralidharan INIT_LIST_HEAD(&netcp->module_head); 200084640e27SKaricheri, Muralidharan INIT_LIST_HEAD(&netcp->txhook_list_head); 200184640e27SKaricheri, Muralidharan INIT_LIST_HEAD(&netcp->rxhook_list_head); 200284640e27SKaricheri, Muralidharan INIT_LIST_HEAD(&netcp->addr_list); 20036a8162e9SMichael Scherban u64_stats_init(&netcp->stats.syncp_rx); 20046a8162e9SMichael Scherban u64_stats_init(&netcp->stats.syncp_tx); 200584640e27SKaricheri, Muralidharan netcp->netcp_device = netcp_device; 200684640e27SKaricheri, Muralidharan netcp->dev = netcp_device->device; 200784640e27SKaricheri, Muralidharan netcp->ndev = ndev; 200884640e27SKaricheri, Muralidharan netcp->ndev_dev = &ndev->dev; 200984640e27SKaricheri, Muralidharan netcp->msg_enable = netif_msg_init(netcp_debug_level, NETCP_DEBUG); 201084640e27SKaricheri, Muralidharan netcp->tx_pause_threshold = MAX_SKB_FRAGS; 201184640e27SKaricheri, Muralidharan netcp->tx_resume_threshold = netcp->tx_pause_threshold; 201284640e27SKaricheri, Muralidharan netcp->node_interface = node_interface; 201384640e27SKaricheri, Muralidharan 201484640e27SKaricheri, Muralidharan ret = of_property_read_u32(node_interface, "efuse-mac", &efuse_mac); 201584640e27SKaricheri, Muralidharan if (efuse_mac) { 201684640e27SKaricheri, Muralidharan if (of_address_to_resource(node, NETCP_EFUSE_REG_INDEX, &res)) { 201784640e27SKaricheri, Muralidharan dev_err(dev, "could not find efuse-mac reg resource\n"); 201884640e27SKaricheri, Muralidharan ret = -ENODEV; 201984640e27SKaricheri, Muralidharan goto quit; 202084640e27SKaricheri, Muralidharan } 202184640e27SKaricheri, Muralidharan size = resource_size(&res); 202284640e27SKaricheri, Muralidharan 202384640e27SKaricheri, Muralidharan if (!devm_request_mem_region(dev, res.start, size, 202484640e27SKaricheri, Muralidharan dev_name(dev))) { 202584640e27SKaricheri, Muralidharan dev_err(dev, "could not reserve resource\n"); 202684640e27SKaricheri, Muralidharan ret = -ENOMEM; 202784640e27SKaricheri, Muralidharan goto quit; 202884640e27SKaricheri, Muralidharan } 202984640e27SKaricheri, Muralidharan 203084640e27SKaricheri, Muralidharan efuse = devm_ioremap_nocache(dev, res.start, size); 203184640e27SKaricheri, Muralidharan if (!efuse) { 203284640e27SKaricheri, Muralidharan dev_err(dev, "could not map resource\n"); 203384640e27SKaricheri, Muralidharan devm_release_mem_region(dev, res.start, size); 203484640e27SKaricheri, Muralidharan ret = -ENOMEM; 203584640e27SKaricheri, Muralidharan goto quit; 203684640e27SKaricheri, Muralidharan } 203784640e27SKaricheri, Muralidharan 203871382bc0SWingMan Kwok emac_arch_get_mac_addr(efuse_mac_addr, efuse, efuse_mac); 203984640e27SKaricheri, Muralidharan if (is_valid_ether_addr(efuse_mac_addr)) 204084640e27SKaricheri, Muralidharan ether_addr_copy(ndev->dev_addr, efuse_mac_addr); 204184640e27SKaricheri, Muralidharan else 20426c1f0a1fSJoe Perches eth_random_addr(ndev->dev_addr); 204384640e27SKaricheri, Muralidharan 204484640e27SKaricheri, Muralidharan devm_iounmap(dev, efuse); 204584640e27SKaricheri, Muralidharan devm_release_mem_region(dev, res.start, size); 204684640e27SKaricheri, Muralidharan } else { 204784640e27SKaricheri, Muralidharan mac_addr = of_get_mac_address(node_interface); 204884640e27SKaricheri, Muralidharan if (mac_addr) 204984640e27SKaricheri, Muralidharan ether_addr_copy(ndev->dev_addr, mac_addr); 205084640e27SKaricheri, Muralidharan else 20516c1f0a1fSJoe Perches eth_random_addr(ndev->dev_addr); 205284640e27SKaricheri, Muralidharan } 205384640e27SKaricheri, Muralidharan 205484640e27SKaricheri, Muralidharan ret = of_property_read_string(node_interface, "rx-channel", 205584640e27SKaricheri, Muralidharan &netcp->dma_chan_name); 205684640e27SKaricheri, Muralidharan if (ret < 0) { 205784640e27SKaricheri, Muralidharan dev_err(dev, "missing \"rx-channel\" parameter\n"); 205884640e27SKaricheri, Muralidharan ret = -ENODEV; 205984640e27SKaricheri, Muralidharan goto quit; 206084640e27SKaricheri, Muralidharan } 206184640e27SKaricheri, Muralidharan 206284640e27SKaricheri, Muralidharan ret = of_property_read_u32(node_interface, "rx-queue", 206384640e27SKaricheri, Muralidharan &netcp->rx_queue_id); 206484640e27SKaricheri, Muralidharan if (ret < 0) { 206584640e27SKaricheri, Muralidharan dev_warn(dev, "missing \"rx-queue\" parameter\n"); 206684640e27SKaricheri, Muralidharan netcp->rx_queue_id = KNAV_QUEUE_QPEND; 206784640e27SKaricheri, Muralidharan } 206884640e27SKaricheri, Muralidharan 206984640e27SKaricheri, Muralidharan ret = of_property_read_u32_array(node_interface, "rx-queue-depth", 207084640e27SKaricheri, Muralidharan netcp->rx_queue_depths, 207184640e27SKaricheri, Muralidharan KNAV_DMA_FDQ_PER_CHAN); 207284640e27SKaricheri, Muralidharan if (ret < 0) { 207384640e27SKaricheri, Muralidharan dev_err(dev, "missing \"rx-queue-depth\" parameter\n"); 207484640e27SKaricheri, Muralidharan netcp->rx_queue_depths[0] = 128; 207584640e27SKaricheri, Muralidharan } 207684640e27SKaricheri, Muralidharan 207784640e27SKaricheri, Muralidharan ret = of_property_read_u32_array(node_interface, "rx-pool", temp, 2); 207884640e27SKaricheri, Muralidharan if (ret < 0) { 207984640e27SKaricheri, Muralidharan dev_err(dev, "missing \"rx-pool\" parameter\n"); 208084640e27SKaricheri, Muralidharan ret = -ENODEV; 208184640e27SKaricheri, Muralidharan goto quit; 208284640e27SKaricheri, Muralidharan } 208384640e27SKaricheri, Muralidharan netcp->rx_pool_size = temp[0]; 208484640e27SKaricheri, Muralidharan netcp->rx_pool_region_id = temp[1]; 208584640e27SKaricheri, Muralidharan 208684640e27SKaricheri, Muralidharan ret = of_property_read_u32_array(node_interface, "tx-pool", temp, 2); 208784640e27SKaricheri, Muralidharan if (ret < 0) { 208884640e27SKaricheri, Muralidharan dev_err(dev, "missing \"tx-pool\" parameter\n"); 208984640e27SKaricheri, Muralidharan ret = -ENODEV; 209084640e27SKaricheri, Muralidharan goto quit; 209184640e27SKaricheri, Muralidharan } 209284640e27SKaricheri, Muralidharan netcp->tx_pool_size = temp[0]; 209384640e27SKaricheri, Muralidharan netcp->tx_pool_region_id = temp[1]; 209484640e27SKaricheri, Muralidharan 209584640e27SKaricheri, Muralidharan if (netcp->tx_pool_size < MAX_SKB_FRAGS) { 209684640e27SKaricheri, Muralidharan dev_err(dev, "tx-pool size too small, must be atleast(%ld)\n", 209784640e27SKaricheri, Muralidharan MAX_SKB_FRAGS); 209884640e27SKaricheri, Muralidharan ret = -ENODEV; 209984640e27SKaricheri, Muralidharan goto quit; 210084640e27SKaricheri, Muralidharan } 210184640e27SKaricheri, Muralidharan 210284640e27SKaricheri, Muralidharan ret = of_property_read_u32(node_interface, "tx-completion-queue", 210384640e27SKaricheri, Muralidharan &netcp->tx_compl_qid); 210484640e27SKaricheri, Muralidharan if (ret < 0) { 210584640e27SKaricheri, Muralidharan dev_warn(dev, "missing \"tx-completion-queue\" parameter\n"); 210684640e27SKaricheri, Muralidharan netcp->tx_compl_qid = KNAV_QUEUE_QPEND; 210784640e27SKaricheri, Muralidharan } 210884640e27SKaricheri, Muralidharan 210984640e27SKaricheri, Muralidharan /* NAPI register */ 211084640e27SKaricheri, Muralidharan netif_napi_add(ndev, &netcp->rx_napi, netcp_rx_poll, NETCP_NAPI_WEIGHT); 2111d64b5e85SEric Dumazet netif_tx_napi_add(ndev, &netcp->tx_napi, netcp_tx_poll, NETCP_NAPI_WEIGHT); 211284640e27SKaricheri, Muralidharan 211384640e27SKaricheri, Muralidharan /* Register the network device */ 211484640e27SKaricheri, Muralidharan ndev->dev_id = 0; 211584640e27SKaricheri, Muralidharan ndev->watchdog_timeo = NETCP_TX_TIMEOUT; 211684640e27SKaricheri, Muralidharan ndev->netdev_ops = &netcp_netdev_ops; 211784640e27SKaricheri, Muralidharan SET_NETDEV_DEV(ndev, dev); 211884640e27SKaricheri, Muralidharan 211984640e27SKaricheri, Muralidharan list_add_tail(&netcp->interface_list, &netcp_device->interface_head); 212084640e27SKaricheri, Muralidharan return 0; 212184640e27SKaricheri, Muralidharan 212284640e27SKaricheri, Muralidharan quit: 212384640e27SKaricheri, Muralidharan free_netdev(ndev); 212484640e27SKaricheri, Muralidharan return ret; 212584640e27SKaricheri, Muralidharan } 212684640e27SKaricheri, Muralidharan 212784640e27SKaricheri, Muralidharan static void netcp_delete_interface(struct netcp_device *netcp_device, 212884640e27SKaricheri, Muralidharan struct net_device *ndev) 212984640e27SKaricheri, Muralidharan { 213084640e27SKaricheri, Muralidharan struct netcp_intf_modpriv *intf_modpriv, *tmp; 213184640e27SKaricheri, Muralidharan struct netcp_intf *netcp = netdev_priv(ndev); 213284640e27SKaricheri, Muralidharan struct netcp_module *module; 213384640e27SKaricheri, Muralidharan 213484640e27SKaricheri, Muralidharan dev_dbg(netcp_device->device, "Removing interface \"%s\"\n", 213584640e27SKaricheri, Muralidharan ndev->name); 213684640e27SKaricheri, Muralidharan 213784640e27SKaricheri, Muralidharan /* Notify each of the modules that the interface is going away */ 213884640e27SKaricheri, Muralidharan list_for_each_entry_safe(intf_modpriv, tmp, &netcp->module_head, 213984640e27SKaricheri, Muralidharan intf_list) { 214084640e27SKaricheri, Muralidharan module = intf_modpriv->netcp_module; 214184640e27SKaricheri, Muralidharan dev_dbg(netcp_device->device, "Releasing module \"%s\"\n", 214284640e27SKaricheri, Muralidharan module->name); 214384640e27SKaricheri, Muralidharan if (module->release) 214484640e27SKaricheri, Muralidharan module->release(intf_modpriv->module_priv); 214584640e27SKaricheri, Muralidharan list_del(&intf_modpriv->intf_list); 214684640e27SKaricheri, Muralidharan } 214784640e27SKaricheri, Muralidharan WARN(!list_empty(&netcp->module_head), "%s interface module list is not empty!\n", 214884640e27SKaricheri, Muralidharan ndev->name); 214984640e27SKaricheri, Muralidharan 215084640e27SKaricheri, Muralidharan list_del(&netcp->interface_list); 215184640e27SKaricheri, Muralidharan 215284640e27SKaricheri, Muralidharan of_node_put(netcp->node_interface); 215384640e27SKaricheri, Muralidharan unregister_netdev(ndev); 215484640e27SKaricheri, Muralidharan free_netdev(ndev); 215584640e27SKaricheri, Muralidharan } 215684640e27SKaricheri, Muralidharan 215784640e27SKaricheri, Muralidharan static int netcp_probe(struct platform_device *pdev) 215884640e27SKaricheri, Muralidharan { 215984640e27SKaricheri, Muralidharan struct device_node *node = pdev->dev.of_node; 216084640e27SKaricheri, Muralidharan struct netcp_intf *netcp_intf, *netcp_tmp; 216184640e27SKaricheri, Muralidharan struct device_node *child, *interfaces; 216284640e27SKaricheri, Muralidharan struct netcp_device *netcp_device; 216384640e27SKaricheri, Muralidharan struct device *dev = &pdev->dev; 2164c52b6782SMurali Karicheri struct netcp_module *module; 216584640e27SKaricheri, Muralidharan int ret; 216684640e27SKaricheri, Muralidharan 216721f706bbSMurali Karicheri if (!knav_dma_device_ready() || 216821f706bbSMurali Karicheri !knav_qmss_device_ready()) 216921f706bbSMurali Karicheri return -EPROBE_DEFER; 217021f706bbSMurali Karicheri 217184640e27SKaricheri, Muralidharan if (!node) { 217284640e27SKaricheri, Muralidharan dev_err(dev, "could not find device info\n"); 217384640e27SKaricheri, Muralidharan return -ENODEV; 217484640e27SKaricheri, Muralidharan } 217584640e27SKaricheri, Muralidharan 217684640e27SKaricheri, Muralidharan /* Allocate a new NETCP device instance */ 217784640e27SKaricheri, Muralidharan netcp_device = devm_kzalloc(dev, sizeof(*netcp_device), GFP_KERNEL); 217884640e27SKaricheri, Muralidharan if (!netcp_device) 217984640e27SKaricheri, Muralidharan return -ENOMEM; 218084640e27SKaricheri, Muralidharan 218184640e27SKaricheri, Muralidharan pm_runtime_enable(&pdev->dev); 218284640e27SKaricheri, Muralidharan ret = pm_runtime_get_sync(&pdev->dev); 218384640e27SKaricheri, Muralidharan if (ret < 0) { 218484640e27SKaricheri, Muralidharan dev_err(dev, "Failed to enable NETCP power-domain\n"); 218584640e27SKaricheri, Muralidharan pm_runtime_disable(&pdev->dev); 218684640e27SKaricheri, Muralidharan return ret; 218784640e27SKaricheri, Muralidharan } 218884640e27SKaricheri, Muralidharan 218984640e27SKaricheri, Muralidharan /* Initialize the NETCP device instance */ 219084640e27SKaricheri, Muralidharan INIT_LIST_HEAD(&netcp_device->interface_head); 219184640e27SKaricheri, Muralidharan INIT_LIST_HEAD(&netcp_device->modpriv_head); 219284640e27SKaricheri, Muralidharan netcp_device->device = dev; 219384640e27SKaricheri, Muralidharan platform_set_drvdata(pdev, netcp_device); 219484640e27SKaricheri, Muralidharan 219584640e27SKaricheri, Muralidharan /* create interfaces */ 219684640e27SKaricheri, Muralidharan interfaces = of_get_child_by_name(node, "netcp-interfaces"); 219784640e27SKaricheri, Muralidharan if (!interfaces) { 219884640e27SKaricheri, Muralidharan dev_err(dev, "could not find netcp-interfaces node\n"); 219984640e27SKaricheri, Muralidharan ret = -ENODEV; 220084640e27SKaricheri, Muralidharan goto probe_quit; 220184640e27SKaricheri, Muralidharan } 220284640e27SKaricheri, Muralidharan 220384640e27SKaricheri, Muralidharan for_each_available_child_of_node(interfaces, child) { 220484640e27SKaricheri, Muralidharan ret = netcp_create_interface(netcp_device, child); 220584640e27SKaricheri, Muralidharan if (ret) { 220621c328dcSRob Herring dev_err(dev, "could not create interface(%pOFn)\n", 220721c328dcSRob Herring child); 220884640e27SKaricheri, Muralidharan goto probe_quit_interface; 220984640e27SKaricheri, Muralidharan } 221084640e27SKaricheri, Muralidharan } 221184640e27SKaricheri, Muralidharan 2212e2897b82SWei Yongjun of_node_put(interfaces); 2213e2897b82SWei Yongjun 221484640e27SKaricheri, Muralidharan /* Add the device instance to the list */ 221584640e27SKaricheri, Muralidharan list_add_tail(&netcp_device->device_list, &netcp_devices); 221684640e27SKaricheri, Muralidharan 2217c52b6782SMurali Karicheri /* Probe & attach any modules already registered */ 2218c52b6782SMurali Karicheri mutex_lock(&netcp_modules_lock); 2219c52b6782SMurali Karicheri for_each_netcp_module(module) { 2220c52b6782SMurali Karicheri ret = netcp_module_probe(netcp_device, module); 2221c52b6782SMurali Karicheri if (ret < 0) 2222c52b6782SMurali Karicheri dev_err(dev, "module(%s) probe failed\n", module->name); 2223c52b6782SMurali Karicheri } 2224c52b6782SMurali Karicheri mutex_unlock(&netcp_modules_lock); 222584640e27SKaricheri, Muralidharan return 0; 222684640e27SKaricheri, Muralidharan 222784640e27SKaricheri, Muralidharan probe_quit_interface: 222884640e27SKaricheri, Muralidharan list_for_each_entry_safe(netcp_intf, netcp_tmp, 222984640e27SKaricheri, Muralidharan &netcp_device->interface_head, 223084640e27SKaricheri, Muralidharan interface_list) { 223184640e27SKaricheri, Muralidharan netcp_delete_interface(netcp_device, netcp_intf->ndev); 223284640e27SKaricheri, Muralidharan } 223384640e27SKaricheri, Muralidharan 2234e2897b82SWei Yongjun of_node_put(interfaces); 2235e2897b82SWei Yongjun 223684640e27SKaricheri, Muralidharan probe_quit: 223784640e27SKaricheri, Muralidharan pm_runtime_put_sync(&pdev->dev); 223884640e27SKaricheri, Muralidharan pm_runtime_disable(&pdev->dev); 223984640e27SKaricheri, Muralidharan platform_set_drvdata(pdev, NULL); 224084640e27SKaricheri, Muralidharan return ret; 224184640e27SKaricheri, Muralidharan } 224284640e27SKaricheri, Muralidharan 224384640e27SKaricheri, Muralidharan static int netcp_remove(struct platform_device *pdev) 224484640e27SKaricheri, Muralidharan { 224584640e27SKaricheri, Muralidharan struct netcp_device *netcp_device = platform_get_drvdata(pdev); 224601a03099SKaricheri, Muralidharan struct netcp_intf *netcp_intf, *netcp_tmp; 224784640e27SKaricheri, Muralidharan struct netcp_inst_modpriv *inst_modpriv, *tmp; 224884640e27SKaricheri, Muralidharan struct netcp_module *module; 224984640e27SKaricheri, Muralidharan 225084640e27SKaricheri, Muralidharan list_for_each_entry_safe(inst_modpriv, tmp, &netcp_device->modpriv_head, 225184640e27SKaricheri, Muralidharan inst_list) { 225284640e27SKaricheri, Muralidharan module = inst_modpriv->netcp_module; 225384640e27SKaricheri, Muralidharan dev_dbg(&pdev->dev, "Removing module \"%s\"\n", module->name); 225484640e27SKaricheri, Muralidharan module->remove(netcp_device, inst_modpriv->module_priv); 225584640e27SKaricheri, Muralidharan list_del(&inst_modpriv->inst_list); 225684640e27SKaricheri, Muralidharan } 225784640e27SKaricheri, Muralidharan 225801a03099SKaricheri, Muralidharan /* now that all modules are removed, clean up the interfaces */ 225901a03099SKaricheri, Muralidharan list_for_each_entry_safe(netcp_intf, netcp_tmp, 226001a03099SKaricheri, Muralidharan &netcp_device->interface_head, 226101a03099SKaricheri, Muralidharan interface_list) { 226201a03099SKaricheri, Muralidharan netcp_delete_interface(netcp_device, netcp_intf->ndev); 226301a03099SKaricheri, Muralidharan } 226401a03099SKaricheri, Muralidharan 226501a03099SKaricheri, Muralidharan WARN(!list_empty(&netcp_device->interface_head), 226601a03099SKaricheri, Muralidharan "%s interface list not empty!\n", pdev->name); 226784640e27SKaricheri, Muralidharan 226884640e27SKaricheri, Muralidharan pm_runtime_put_sync(&pdev->dev); 226984640e27SKaricheri, Muralidharan pm_runtime_disable(&pdev->dev); 227084640e27SKaricheri, Muralidharan platform_set_drvdata(pdev, NULL); 227184640e27SKaricheri, Muralidharan return 0; 227284640e27SKaricheri, Muralidharan } 227384640e27SKaricheri, Muralidharan 22741156c965SFabian Frederick static const struct of_device_id of_match[] = { 227584640e27SKaricheri, Muralidharan { .compatible = "ti,netcp-1.0", }, 227684640e27SKaricheri, Muralidharan {}, 227784640e27SKaricheri, Muralidharan }; 227884640e27SKaricheri, Muralidharan MODULE_DEVICE_TABLE(of, of_match); 227984640e27SKaricheri, Muralidharan 228084640e27SKaricheri, Muralidharan static struct platform_driver netcp_driver = { 228184640e27SKaricheri, Muralidharan .driver = { 228284640e27SKaricheri, Muralidharan .name = "netcp-1.0", 228384640e27SKaricheri, Muralidharan .of_match_table = of_match, 228484640e27SKaricheri, Muralidharan }, 228584640e27SKaricheri, Muralidharan .probe = netcp_probe, 228684640e27SKaricheri, Muralidharan .remove = netcp_remove, 228784640e27SKaricheri, Muralidharan }; 228884640e27SKaricheri, Muralidharan module_platform_driver(netcp_driver); 228984640e27SKaricheri, Muralidharan 229084640e27SKaricheri, Muralidharan MODULE_LICENSE("GPL v2"); 229184640e27SKaricheri, Muralidharan MODULE_DESCRIPTION("TI NETCP driver for Keystone SOCs"); 229284640e27SKaricheri, Muralidharan MODULE_AUTHOR("Sandeep Nair <sandeep_n@ti.com"); 2293