12874c5fdSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-or-later 2f2f23566SVivien Didelot /* 3f2f23566SVivien Didelot * Handling of a master device, switching frames via its switch fabric CPU port 4f2f23566SVivien Didelot * 5f2f23566SVivien Didelot * Copyright (c) 2017 Savoir-faire Linux Inc. 6f2f23566SVivien Didelot * Vivien Didelot <vivien.didelot@savoirfairelinux.com> 7f2f23566SVivien Didelot */ 8f2f23566SVivien Didelot 9f2f23566SVivien Didelot #include "dsa_priv.h" 10f2f23566SVivien Didelot 1148e23311SVivien Didelot static int dsa_master_get_regs_len(struct net_device *dev) 1248e23311SVivien Didelot { 1348e23311SVivien Didelot struct dsa_port *cpu_dp = dev->dsa_ptr; 1448e23311SVivien Didelot const struct ethtool_ops *ops = cpu_dp->orig_ethtool_ops; 1548e23311SVivien Didelot struct dsa_switch *ds = cpu_dp->ds; 1648e23311SVivien Didelot int port = cpu_dp->index; 1748e23311SVivien Didelot int ret = 0; 1848e23311SVivien Didelot int len; 1948e23311SVivien Didelot 2048e23311SVivien Didelot if (ops->get_regs_len) { 2148e23311SVivien Didelot len = ops->get_regs_len(dev); 2248e23311SVivien Didelot if (len < 0) 2348e23311SVivien Didelot return len; 2448e23311SVivien Didelot ret += len; 2548e23311SVivien Didelot } 2648e23311SVivien Didelot 2748e23311SVivien Didelot ret += sizeof(struct ethtool_drvinfo); 2848e23311SVivien Didelot ret += sizeof(struct ethtool_regs); 2948e23311SVivien Didelot 3048e23311SVivien Didelot if (ds->ops->get_regs_len) { 3148e23311SVivien Didelot len = ds->ops->get_regs_len(ds, port); 3248e23311SVivien Didelot if (len < 0) 3348e23311SVivien Didelot return len; 3448e23311SVivien Didelot ret += len; 3548e23311SVivien Didelot } 3648e23311SVivien Didelot 3748e23311SVivien Didelot return ret; 3848e23311SVivien Didelot } 3948e23311SVivien Didelot 4048e23311SVivien Didelot static void dsa_master_get_regs(struct net_device *dev, 4148e23311SVivien Didelot struct ethtool_regs *regs, void *data) 4248e23311SVivien Didelot { 4348e23311SVivien Didelot struct dsa_port *cpu_dp = dev->dsa_ptr; 4448e23311SVivien Didelot const struct ethtool_ops *ops = cpu_dp->orig_ethtool_ops; 4548e23311SVivien Didelot struct dsa_switch *ds = cpu_dp->ds; 4648e23311SVivien Didelot struct ethtool_drvinfo *cpu_info; 4748e23311SVivien Didelot struct ethtool_regs *cpu_regs; 4848e23311SVivien Didelot int port = cpu_dp->index; 4948e23311SVivien Didelot int len; 5048e23311SVivien Didelot 5148e23311SVivien Didelot if (ops->get_regs_len && ops->get_regs) { 5248e23311SVivien Didelot len = ops->get_regs_len(dev); 5348e23311SVivien Didelot if (len < 0) 5448e23311SVivien Didelot return; 5548e23311SVivien Didelot regs->len = len; 5648e23311SVivien Didelot ops->get_regs(dev, regs, data); 5748e23311SVivien Didelot data += regs->len; 5848e23311SVivien Didelot } 5948e23311SVivien Didelot 6048e23311SVivien Didelot cpu_info = (struct ethtool_drvinfo *)data; 6148e23311SVivien Didelot strlcpy(cpu_info->driver, "dsa", sizeof(cpu_info->driver)); 6248e23311SVivien Didelot data += sizeof(*cpu_info); 6348e23311SVivien Didelot cpu_regs = (struct ethtool_regs *)data; 6448e23311SVivien Didelot data += sizeof(*cpu_regs); 6548e23311SVivien Didelot 6648e23311SVivien Didelot if (ds->ops->get_regs_len && ds->ops->get_regs) { 6748e23311SVivien Didelot len = ds->ops->get_regs_len(ds, port); 6848e23311SVivien Didelot if (len < 0) 6948e23311SVivien Didelot return; 7048e23311SVivien Didelot cpu_regs->len = len; 7148e23311SVivien Didelot ds->ops->get_regs(ds, port, cpu_regs, data); 7248e23311SVivien Didelot } 7348e23311SVivien Didelot } 7448e23311SVivien Didelot 75f2f23566SVivien Didelot static void dsa_master_get_ethtool_stats(struct net_device *dev, 76f2f23566SVivien Didelot struct ethtool_stats *stats, 77f2f23566SVivien Didelot uint64_t *data) 78f2f23566SVivien Didelot { 792f657a60SVivien Didelot struct dsa_port *cpu_dp = dev->dsa_ptr; 807ec764eeSVivien Didelot const struct ethtool_ops *ops = cpu_dp->orig_ethtool_ops; 817ec764eeSVivien Didelot struct dsa_switch *ds = cpu_dp->ds; 827ec764eeSVivien Didelot int port = cpu_dp->index; 83f2f23566SVivien Didelot int count = 0; 84f2f23566SVivien Didelot 851d1e79f1SFlorian Fainelli if (ops->get_sset_count && ops->get_ethtool_stats) { 86f2f23566SVivien Didelot count = ops->get_sset_count(dev, ETH_SS_STATS); 87f2f23566SVivien Didelot ops->get_ethtool_stats(dev, stats, data); 88f2f23566SVivien Didelot } 89f2f23566SVivien Didelot 90f2f23566SVivien Didelot if (ds->ops->get_ethtool_stats) 917ec764eeSVivien Didelot ds->ops->get_ethtool_stats(ds, port, data + count); 92f2f23566SVivien Didelot } 93f2f23566SVivien Didelot 94cf963573SFlorian Fainelli static void dsa_master_get_ethtool_phy_stats(struct net_device *dev, 95cf963573SFlorian Fainelli struct ethtool_stats *stats, 96cf963573SFlorian Fainelli uint64_t *data) 97cf963573SFlorian Fainelli { 98cf963573SFlorian Fainelli struct dsa_port *cpu_dp = dev->dsa_ptr; 99cf963573SFlorian Fainelli const struct ethtool_ops *ops = cpu_dp->orig_ethtool_ops; 100cf963573SFlorian Fainelli struct dsa_switch *ds = cpu_dp->ds; 101cf963573SFlorian Fainelli int port = cpu_dp->index; 102cf963573SFlorian Fainelli int count = 0; 103cf963573SFlorian Fainelli 104cf963573SFlorian Fainelli if (dev->phydev && !ops->get_ethtool_phy_stats) { 105cf963573SFlorian Fainelli count = phy_ethtool_get_sset_count(dev->phydev); 106cf963573SFlorian Fainelli if (count >= 0) 107cf963573SFlorian Fainelli phy_ethtool_get_stats(dev->phydev, stats, data); 108cf963573SFlorian Fainelli } else if (ops->get_sset_count && ops->get_ethtool_phy_stats) { 109cf963573SFlorian Fainelli count = ops->get_sset_count(dev, ETH_SS_PHY_STATS); 110cf963573SFlorian Fainelli ops->get_ethtool_phy_stats(dev, stats, data); 111cf963573SFlorian Fainelli } 112cf963573SFlorian Fainelli 113cf963573SFlorian Fainelli if (count < 0) 114cf963573SFlorian Fainelli count = 0; 115cf963573SFlorian Fainelli 116cf963573SFlorian Fainelli if (ds->ops->get_ethtool_phy_stats) 117cf963573SFlorian Fainelli ds->ops->get_ethtool_phy_stats(ds, port, data + count); 118cf963573SFlorian Fainelli } 119cf963573SFlorian Fainelli 120f2f23566SVivien Didelot static int dsa_master_get_sset_count(struct net_device *dev, int sset) 121f2f23566SVivien Didelot { 1222f657a60SVivien Didelot struct dsa_port *cpu_dp = dev->dsa_ptr; 1237ec764eeSVivien Didelot const struct ethtool_ops *ops = cpu_dp->orig_ethtool_ops; 1247ec764eeSVivien Didelot struct dsa_switch *ds = cpu_dp->ds; 125f2f23566SVivien Didelot int count = 0; 126f2f23566SVivien Didelot 127cf963573SFlorian Fainelli if (sset == ETH_SS_PHY_STATS && dev->phydev && 128cf963573SFlorian Fainelli !ops->get_ethtool_phy_stats) 129cf963573SFlorian Fainelli count = phy_ethtool_get_sset_count(dev->phydev); 130cf963573SFlorian Fainelli else if (ops->get_sset_count) 13189f09048SFlorian Fainelli count = ops->get_sset_count(dev, sset); 132cf963573SFlorian Fainelli 13389f09048SFlorian Fainelli if (count < 0) 13489f09048SFlorian Fainelli count = 0; 135f2f23566SVivien Didelot 13689f09048SFlorian Fainelli if (ds->ops->get_sset_count) 13789f09048SFlorian Fainelli count += ds->ops->get_sset_count(ds, cpu_dp->index, sset); 138f2f23566SVivien Didelot 139f2f23566SVivien Didelot return count; 140f2f23566SVivien Didelot } 141f2f23566SVivien Didelot 142f2f23566SVivien Didelot static void dsa_master_get_strings(struct net_device *dev, uint32_t stringset, 143f2f23566SVivien Didelot uint8_t *data) 144f2f23566SVivien Didelot { 1452f657a60SVivien Didelot struct dsa_port *cpu_dp = dev->dsa_ptr; 1467ec764eeSVivien Didelot const struct ethtool_ops *ops = cpu_dp->orig_ethtool_ops; 1477ec764eeSVivien Didelot struct dsa_switch *ds = cpu_dp->ds; 1487ec764eeSVivien Didelot int port = cpu_dp->index; 149f2f23566SVivien Didelot int len = ETH_GSTRING_LEN; 150f2f23566SVivien Didelot int mcount = 0, count; 151f2f23566SVivien Didelot unsigned int i; 152f2f23566SVivien Didelot uint8_t pfx[4]; 153f2f23566SVivien Didelot uint8_t *ndata; 154f2f23566SVivien Didelot 1557ec764eeSVivien Didelot snprintf(pfx, sizeof(pfx), "p%.2d", port); 156f2f23566SVivien Didelot /* We do not want to be NULL-terminated, since this is a prefix */ 157f2f23566SVivien Didelot pfx[sizeof(pfx) - 1] = '_'; 158f2f23566SVivien Didelot 159cf963573SFlorian Fainelli if (stringset == ETH_SS_PHY_STATS && dev->phydev && 160cf963573SFlorian Fainelli !ops->get_ethtool_phy_stats) { 161cf963573SFlorian Fainelli mcount = phy_ethtool_get_sset_count(dev->phydev); 162cf963573SFlorian Fainelli if (mcount < 0) 163cf963573SFlorian Fainelli mcount = 0; 164cf963573SFlorian Fainelli else 165cf963573SFlorian Fainelli phy_ethtool_get_strings(dev->phydev, data); 166cf963573SFlorian Fainelli } else if (ops->get_sset_count && ops->get_strings) { 16789f09048SFlorian Fainelli mcount = ops->get_sset_count(dev, stringset); 16889f09048SFlorian Fainelli if (mcount < 0) 16989f09048SFlorian Fainelli mcount = 0; 170f2f23566SVivien Didelot ops->get_strings(dev, stringset, data); 171f2f23566SVivien Didelot } 172f2f23566SVivien Didelot 17389f09048SFlorian Fainelli if (ds->ops->get_strings) { 174f2f23566SVivien Didelot ndata = data + mcount * len; 175f2f23566SVivien Didelot /* This function copies ETH_GSTRINGS_LEN bytes, we will mangle 176f2f23566SVivien Didelot * the output after to prepend our CPU port prefix we 177f2f23566SVivien Didelot * constructed earlier 178f2f23566SVivien Didelot */ 17989f09048SFlorian Fainelli ds->ops->get_strings(ds, port, stringset, ndata); 18089f09048SFlorian Fainelli count = ds->ops->get_sset_count(ds, port, stringset); 181f2f23566SVivien Didelot for (i = 0; i < count; i++) { 182f2f23566SVivien Didelot memmove(ndata + (i * len + sizeof(pfx)), 183f2f23566SVivien Didelot ndata + i * len, len - sizeof(pfx)); 184f2f23566SVivien Didelot memcpy(ndata + i * len, pfx, sizeof(pfx)); 185f2f23566SVivien Didelot } 186f2f23566SVivien Didelot } 187f2f23566SVivien Didelot } 188f2f23566SVivien Didelot 189f685e609SVladimir Oltean static int dsa_master_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd) 190f685e609SVladimir Oltean { 191f685e609SVladimir Oltean struct dsa_port *cpu_dp = dev->dsa_ptr; 192f685e609SVladimir Oltean struct dsa_switch *ds = cpu_dp->ds; 193f685e609SVladimir Oltean struct dsa_switch_tree *dst; 194f685e609SVladimir Oltean int err = -EOPNOTSUPP; 195f685e609SVladimir Oltean struct dsa_port *dp; 196f685e609SVladimir Oltean 197f685e609SVladimir Oltean dst = ds->dst; 198f685e609SVladimir Oltean 199f685e609SVladimir Oltean switch (cmd) { 200f685e609SVladimir Oltean case SIOCGHWTSTAMP: 201f685e609SVladimir Oltean case SIOCSHWTSTAMP: 202f685e609SVladimir Oltean /* Deny PTP operations on master if there is at least one 203f685e609SVladimir Oltean * switch in the tree that is PTP capable. 204f685e609SVladimir Oltean */ 205f685e609SVladimir Oltean list_for_each_entry(dp, &dst->ports, list) 206f685e609SVladimir Oltean if (dp->ds->ops->port_hwtstamp_get || 207f685e609SVladimir Oltean dp->ds->ops->port_hwtstamp_set) 208f685e609SVladimir Oltean return -EBUSY; 209f685e609SVladimir Oltean break; 210f685e609SVladimir Oltean } 211f685e609SVladimir Oltean 2129c0c7014SFlorian Fainelli if (dev->netdev_ops->ndo_do_ioctl) 2139c0c7014SFlorian Fainelli err = dev->netdev_ops->ndo_do_ioctl(dev, ifr, cmd); 214f685e609SVladimir Oltean 215f685e609SVladimir Oltean return err; 216f685e609SVladimir Oltean } 217f685e609SVladimir Oltean 2189c0c7014SFlorian Fainelli static const struct dsa_netdevice_ops dsa_netdev_ops = { 2199c0c7014SFlorian Fainelli .ndo_do_ioctl = dsa_master_ioctl, 2209c0c7014SFlorian Fainelli }; 2219c0c7014SFlorian Fainelli 22217a22fcfSVivien Didelot static int dsa_master_ethtool_setup(struct net_device *dev) 223f2f23566SVivien Didelot { 2242f657a60SVivien Didelot struct dsa_port *cpu_dp = dev->dsa_ptr; 2257ec764eeSVivien Didelot struct dsa_switch *ds = cpu_dp->ds; 226f2f23566SVivien Didelot struct ethtool_ops *ops; 227f2f23566SVivien Didelot 228f2f23566SVivien Didelot ops = devm_kzalloc(ds->dev, sizeof(*ops), GFP_KERNEL); 229f2f23566SVivien Didelot if (!ops) 230f2f23566SVivien Didelot return -ENOMEM; 231f2f23566SVivien Didelot 2327ec764eeSVivien Didelot cpu_dp->orig_ethtool_ops = dev->ethtool_ops; 2337ec764eeSVivien Didelot if (cpu_dp->orig_ethtool_ops) 2347ec764eeSVivien Didelot memcpy(ops, cpu_dp->orig_ethtool_ops, sizeof(*ops)); 235f2f23566SVivien Didelot 23648e23311SVivien Didelot ops->get_regs_len = dsa_master_get_regs_len; 23748e23311SVivien Didelot ops->get_regs = dsa_master_get_regs; 238f2f23566SVivien Didelot ops->get_sset_count = dsa_master_get_sset_count; 239f2f23566SVivien Didelot ops->get_ethtool_stats = dsa_master_get_ethtool_stats; 240f2f23566SVivien Didelot ops->get_strings = dsa_master_get_strings; 241cf963573SFlorian Fainelli ops->get_ethtool_phy_stats = dsa_master_get_ethtool_phy_stats; 242f2f23566SVivien Didelot 243f2f23566SVivien Didelot dev->ethtool_ops = ops; 244f2f23566SVivien Didelot 245f2f23566SVivien Didelot return 0; 246f2f23566SVivien Didelot } 247f2f23566SVivien Didelot 24817a22fcfSVivien Didelot static void dsa_master_ethtool_teardown(struct net_device *dev) 249f2f23566SVivien Didelot { 2502f657a60SVivien Didelot struct dsa_port *cpu_dp = dev->dsa_ptr; 251f2f23566SVivien Didelot 2527ec764eeSVivien Didelot dev->ethtool_ops = cpu_dp->orig_ethtool_ops; 2537ec764eeSVivien Didelot cpu_dp->orig_ethtool_ops = NULL; 254f2f23566SVivien Didelot } 25517a22fcfSVivien Didelot 2569c0c7014SFlorian Fainelli static void dsa_netdev_ops_set(struct net_device *dev, 2579c0c7014SFlorian Fainelli const struct dsa_netdevice_ops *ops) 258da7b9e9bSFlorian Fainelli { 2599c0c7014SFlorian Fainelli dev->dsa_ptr->netdev_ops = ops; 260da7b9e9bSFlorian Fainelli } 261da7b9e9bSFlorian Fainelli 262*c3975400SVladimir Oltean static void dsa_master_set_promiscuity(struct net_device *dev, int inc) 263*c3975400SVladimir Oltean { 264*c3975400SVladimir Oltean const struct dsa_device_ops *ops = dev->dsa_ptr->tag_ops; 265*c3975400SVladimir Oltean 266*c3975400SVladimir Oltean if (!ops->promisc_on_master) 267*c3975400SVladimir Oltean return; 268*c3975400SVladimir Oltean 269*c3975400SVladimir Oltean rtnl_lock(); 270*c3975400SVladimir Oltean dev_set_promiscuity(dev, inc); 271*c3975400SVladimir Oltean rtnl_unlock(); 272*c3975400SVladimir Oltean } 273*c3975400SVladimir Oltean 274a3d7e01dSFlorian Fainelli static ssize_t tagging_show(struct device *d, struct device_attribute *attr, 275a3d7e01dSFlorian Fainelli char *buf) 276a3d7e01dSFlorian Fainelli { 277a3d7e01dSFlorian Fainelli struct net_device *dev = to_net_dev(d); 278a3d7e01dSFlorian Fainelli struct dsa_port *cpu_dp = dev->dsa_ptr; 279a3d7e01dSFlorian Fainelli 280a3d7e01dSFlorian Fainelli return sprintf(buf, "%s\n", 281a3d7e01dSFlorian Fainelli dsa_tag_protocol_to_str(cpu_dp->tag_ops)); 282a3d7e01dSFlorian Fainelli } 283a3d7e01dSFlorian Fainelli static DEVICE_ATTR_RO(tagging); 284a3d7e01dSFlorian Fainelli 285a3d7e01dSFlorian Fainelli static struct attribute *dsa_slave_attrs[] = { 286a3d7e01dSFlorian Fainelli &dev_attr_tagging.attr, 287a3d7e01dSFlorian Fainelli NULL 288a3d7e01dSFlorian Fainelli }; 289a3d7e01dSFlorian Fainelli 290a3d7e01dSFlorian Fainelli static const struct attribute_group dsa_group = { 291a3d7e01dSFlorian Fainelli .name = "dsa", 292a3d7e01dSFlorian Fainelli .attrs = dsa_slave_attrs, 293a3d7e01dSFlorian Fainelli }; 294a3d7e01dSFlorian Fainelli 29591ba4795SAndrew Lunn static void dsa_master_reset_mtu(struct net_device *dev) 29691ba4795SAndrew Lunn { 29791ba4795SAndrew Lunn int err; 29891ba4795SAndrew Lunn 29991ba4795SAndrew Lunn rtnl_lock(); 30091ba4795SAndrew Lunn err = dev_set_mtu(dev, ETH_DATA_LEN); 30191ba4795SAndrew Lunn if (err) 30291ba4795SAndrew Lunn netdev_dbg(dev, 30391ba4795SAndrew Lunn "Unable to reset MTU to exclude DSA overheads\n"); 30491ba4795SAndrew Lunn rtnl_unlock(); 30591ba4795SAndrew Lunn } 30691ba4795SAndrew Lunn 307845e0ebbSCong Wang static struct lock_class_key dsa_master_addr_list_lock_key; 308845e0ebbSCong Wang 30917a22fcfSVivien Didelot int dsa_master_setup(struct net_device *dev, struct dsa_port *cpu_dp) 31017a22fcfSVivien Didelot { 311a3d7e01dSFlorian Fainelli int ret; 312a3d7e01dSFlorian Fainelli 313bfcb8132SVladimir Oltean rtnl_lock(); 314bfcb8132SVladimir Oltean ret = dev_set_mtu(dev, ETH_DATA_LEN + cpu_dp->tag_ops->overhead); 315bfcb8132SVladimir Oltean rtnl_unlock(); 316bfcb8132SVladimir Oltean if (ret) 317bfcb8132SVladimir Oltean netdev_warn(dev, "error %d setting MTU to include DSA overhead\n", 318bfcb8132SVladimir Oltean ret); 319dc0fe7d4SAndrew Lunn 32017a22fcfSVivien Didelot /* If we use a tagging format that doesn't have an ethertype 32117a22fcfSVivien Didelot * field, make sure that all packets from this point on get 32217a22fcfSVivien Didelot * sent to the tag format's receive function. 32317a22fcfSVivien Didelot */ 32417a22fcfSVivien Didelot wmb(); 32517a22fcfSVivien Didelot 32617a22fcfSVivien Didelot dev->dsa_ptr = cpu_dp; 327845e0ebbSCong Wang lockdep_set_class(&dev->addr_list_lock, 328845e0ebbSCong Wang &dsa_master_addr_list_lock_key); 329*c3975400SVladimir Oltean 330*c3975400SVladimir Oltean dsa_master_set_promiscuity(dev, 1); 331*c3975400SVladimir Oltean 332a3d7e01dSFlorian Fainelli ret = dsa_master_ethtool_setup(dev); 333a3d7e01dSFlorian Fainelli if (ret) 334*c3975400SVladimir Oltean goto out_err_reset_promisc; 335a3d7e01dSFlorian Fainelli 3369c0c7014SFlorian Fainelli dsa_netdev_ops_set(dev, &dsa_netdev_ops); 337da7b9e9bSFlorian Fainelli 338a3d7e01dSFlorian Fainelli ret = sysfs_create_group(&dev->dev.kobj, &dsa_group); 339a3d7e01dSFlorian Fainelli if (ret) 340da7b9e9bSFlorian Fainelli goto out_err_ndo_teardown; 341a3d7e01dSFlorian Fainelli 342a3d7e01dSFlorian Fainelli return ret; 343da7b9e9bSFlorian Fainelli 344da7b9e9bSFlorian Fainelli out_err_ndo_teardown: 3459c0c7014SFlorian Fainelli dsa_netdev_ops_set(dev, NULL); 346da7b9e9bSFlorian Fainelli dsa_master_ethtool_teardown(dev); 347*c3975400SVladimir Oltean out_err_reset_promisc: 348*c3975400SVladimir Oltean dsa_master_set_promiscuity(dev, -1); 349da7b9e9bSFlorian Fainelli return ret; 35017a22fcfSVivien Didelot } 35117a22fcfSVivien Didelot 35217a22fcfSVivien Didelot void dsa_master_teardown(struct net_device *dev) 35317a22fcfSVivien Didelot { 354a3d7e01dSFlorian Fainelli sysfs_remove_group(&dev->dev.kobj, &dsa_group); 3559c0c7014SFlorian Fainelli dsa_netdev_ops_set(dev, NULL); 35617a22fcfSVivien Didelot dsa_master_ethtool_teardown(dev); 35791ba4795SAndrew Lunn dsa_master_reset_mtu(dev); 358*c3975400SVladimir Oltean dsa_master_set_promiscuity(dev, -1); 35917a22fcfSVivien Didelot 36017a22fcfSVivien Didelot dev->dsa_ptr = NULL; 36117a22fcfSVivien Didelot 36217a22fcfSVivien Didelot /* If we used a tagging format that doesn't have an ethertype 36317a22fcfSVivien Didelot * field, make sure that all packets from this point get sent 36417a22fcfSVivien Didelot * without the tag and go through the regular receive path. 36517a22fcfSVivien Didelot */ 36617a22fcfSVivien Didelot wmb(); 36717a22fcfSVivien Didelot } 368