1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Intel IFC VF NIC driver for virtio dataplane offloading 4 * 5 * Copyright (C) 2020 Intel Corporation. 6 * 7 * Author: Zhu Lingshan <lingshan.zhu@intel.com> 8 * 9 */ 10 11 #include "ifcvf_base.h" 12 13 u16 ifcvf_set_vq_vector(struct ifcvf_hw *hw, u16 qid, int vector) 14 { 15 struct virtio_pci_common_cfg __iomem *cfg = hw->common_cfg; 16 17 vp_iowrite16(qid, &cfg->queue_select); 18 vp_iowrite16(vector, &cfg->queue_msix_vector); 19 20 return vp_ioread16(&cfg->queue_msix_vector); 21 } 22 23 u16 ifcvf_set_config_vector(struct ifcvf_hw *hw, int vector) 24 { 25 struct virtio_pci_common_cfg __iomem *cfg = hw->common_cfg; 26 27 vp_iowrite16(vector, &cfg->msix_config); 28 29 return vp_ioread16(&cfg->msix_config); 30 } 31 32 static void __iomem *get_cap_addr(struct ifcvf_hw *hw, 33 struct virtio_pci_cap *cap) 34 { 35 u32 length, offset; 36 u8 bar; 37 38 length = le32_to_cpu(cap->length); 39 offset = le32_to_cpu(cap->offset); 40 bar = cap->bar; 41 42 if (bar >= IFCVF_PCI_MAX_RESOURCE) { 43 IFCVF_DBG(hw->pdev, 44 "Invalid bar number %u to get capabilities\n", bar); 45 return NULL; 46 } 47 48 if (offset + length > pci_resource_len(hw->pdev, bar)) { 49 IFCVF_DBG(hw->pdev, 50 "offset(%u) + len(%u) overflows bar%u's capability\n", 51 offset, length, bar); 52 return NULL; 53 } 54 55 return hw->base[bar] + offset; 56 } 57 58 static int ifcvf_read_config_range(struct pci_dev *dev, 59 uint32_t *val, int size, int where) 60 { 61 int ret, i; 62 63 for (i = 0; i < size; i += 4) { 64 ret = pci_read_config_dword(dev, where + i, val + i / 4); 65 if (ret < 0) 66 return ret; 67 } 68 69 return 0; 70 } 71 72 int ifcvf_init_hw(struct ifcvf_hw *hw, struct pci_dev *pdev) 73 { 74 struct virtio_pci_cap cap; 75 u16 notify_off; 76 int ret; 77 u8 pos; 78 u32 i; 79 80 ret = pci_read_config_byte(pdev, PCI_CAPABILITY_LIST, &pos); 81 if (ret < 0) { 82 IFCVF_ERR(pdev, "Failed to read PCI capability list\n"); 83 return -EIO; 84 } 85 hw->pdev = pdev; 86 87 while (pos) { 88 ret = ifcvf_read_config_range(pdev, (u32 *)&cap, 89 sizeof(cap), pos); 90 if (ret < 0) { 91 IFCVF_ERR(pdev, 92 "Failed to get PCI capability at %x\n", pos); 93 break; 94 } 95 96 if (cap.cap_vndr != PCI_CAP_ID_VNDR) 97 goto next; 98 99 switch (cap.cfg_type) { 100 case VIRTIO_PCI_CAP_COMMON_CFG: 101 hw->common_cfg = get_cap_addr(hw, &cap); 102 IFCVF_DBG(pdev, "hw->common_cfg = %p\n", 103 hw->common_cfg); 104 break; 105 case VIRTIO_PCI_CAP_NOTIFY_CFG: 106 pci_read_config_dword(pdev, pos + sizeof(cap), 107 &hw->notify_off_multiplier); 108 hw->notify_bar = cap.bar; 109 hw->notify_base = get_cap_addr(hw, &cap); 110 hw->notify_base_pa = pci_resource_start(pdev, cap.bar) + 111 le32_to_cpu(cap.offset); 112 IFCVF_DBG(pdev, "hw->notify_base = %p\n", 113 hw->notify_base); 114 break; 115 case VIRTIO_PCI_CAP_ISR_CFG: 116 hw->isr = get_cap_addr(hw, &cap); 117 IFCVF_DBG(pdev, "hw->isr = %p\n", hw->isr); 118 break; 119 case VIRTIO_PCI_CAP_DEVICE_CFG: 120 hw->dev_cfg = get_cap_addr(hw, &cap); 121 hw->cap_dev_config_size = le32_to_cpu(cap.length); 122 IFCVF_DBG(pdev, "hw->dev_cfg = %p\n", hw->dev_cfg); 123 break; 124 } 125 126 next: 127 pos = cap.cap_next; 128 } 129 130 if (hw->common_cfg == NULL || hw->notify_base == NULL || 131 hw->isr == NULL || hw->dev_cfg == NULL) { 132 IFCVF_ERR(pdev, "Incomplete PCI capabilities\n"); 133 return -EIO; 134 } 135 136 hw->nr_vring = vp_ioread16(&hw->common_cfg->num_queues); 137 138 for (i = 0; i < hw->nr_vring; i++) { 139 vp_iowrite16(i, &hw->common_cfg->queue_select); 140 notify_off = vp_ioread16(&hw->common_cfg->queue_notify_off); 141 hw->vring[i].notify_addr = hw->notify_base + 142 notify_off * hw->notify_off_multiplier; 143 hw->vring[i].notify_pa = hw->notify_base_pa + 144 notify_off * hw->notify_off_multiplier; 145 hw->vring[i].irq = -EINVAL; 146 } 147 148 hw->lm_cfg = hw->base[IFCVF_LM_BAR]; 149 150 IFCVF_DBG(pdev, 151 "PCI capability mapping: common cfg: %p, notify base: %p\n, isr cfg: %p, device cfg: %p, multiplier: %u\n", 152 hw->common_cfg, hw->notify_base, hw->isr, 153 hw->dev_cfg, hw->notify_off_multiplier); 154 155 hw->vqs_reused_irq = -EINVAL; 156 hw->config_irq = -EINVAL; 157 158 return 0; 159 } 160 161 u8 ifcvf_get_status(struct ifcvf_hw *hw) 162 { 163 return vp_ioread8(&hw->common_cfg->device_status); 164 } 165 166 void ifcvf_set_status(struct ifcvf_hw *hw, u8 status) 167 { 168 vp_iowrite8(status, &hw->common_cfg->device_status); 169 } 170 171 void ifcvf_reset(struct ifcvf_hw *hw) 172 { 173 hw->config_cb.callback = NULL; 174 hw->config_cb.private = NULL; 175 176 ifcvf_set_status(hw, 0); 177 /* flush set_status, make sure VF is stopped, reset */ 178 ifcvf_get_status(hw); 179 } 180 181 static void ifcvf_add_status(struct ifcvf_hw *hw, u8 status) 182 { 183 if (status != 0) 184 status |= ifcvf_get_status(hw); 185 186 ifcvf_set_status(hw, status); 187 ifcvf_get_status(hw); 188 } 189 190 u64 ifcvf_get_hw_features(struct ifcvf_hw *hw) 191 { 192 struct virtio_pci_common_cfg __iomem *cfg = hw->common_cfg; 193 u32 features_lo, features_hi; 194 u64 features; 195 196 vp_iowrite32(0, &cfg->device_feature_select); 197 features_lo = vp_ioread32(&cfg->device_feature); 198 199 vp_iowrite32(1, &cfg->device_feature_select); 200 features_hi = vp_ioread32(&cfg->device_feature); 201 202 features = ((u64)features_hi << 32) | features_lo; 203 204 return features; 205 } 206 207 u64 ifcvf_get_features(struct ifcvf_hw *hw) 208 { 209 return hw->dev_features; 210 } 211 212 int ifcvf_verify_min_features(struct ifcvf_hw *hw, u64 features) 213 { 214 if (!(features & BIT_ULL(VIRTIO_F_ACCESS_PLATFORM)) && features) { 215 IFCVF_ERR(hw->pdev, "VIRTIO_F_ACCESS_PLATFORM is not negotiated\n"); 216 return -EINVAL; 217 } 218 219 return 0; 220 } 221 222 u32 ifcvf_get_config_size(struct ifcvf_hw *hw) 223 { 224 u32 net_config_size = sizeof(struct virtio_net_config); 225 u32 blk_config_size = sizeof(struct virtio_blk_config); 226 u32 cap_size = hw->cap_dev_config_size; 227 u32 config_size; 228 229 /* If the onboard device config space size is greater than 230 * the size of struct virtio_net/blk_config, only the spec 231 * implementing contents size is returned, this is very 232 * unlikely, defensive programming. 233 */ 234 switch (hw->dev_type) { 235 case VIRTIO_ID_NET: 236 config_size = min(cap_size, net_config_size); 237 break; 238 case VIRTIO_ID_BLOCK: 239 config_size = min(cap_size, blk_config_size); 240 break; 241 default: 242 config_size = 0; 243 IFCVF_ERR(hw->pdev, "VIRTIO ID %u not supported\n", hw->dev_type); 244 } 245 246 return config_size; 247 } 248 249 void ifcvf_read_dev_config(struct ifcvf_hw *hw, u64 offset, 250 void *dst, int length) 251 { 252 u8 old_gen, new_gen, *p; 253 int i; 254 255 WARN_ON(offset + length > hw->config_size); 256 do { 257 old_gen = vp_ioread8(&hw->common_cfg->config_generation); 258 p = dst; 259 for (i = 0; i < length; i++) 260 *p++ = vp_ioread8(hw->dev_cfg + offset + i); 261 262 new_gen = vp_ioread8(&hw->common_cfg->config_generation); 263 } while (old_gen != new_gen); 264 } 265 266 void ifcvf_write_dev_config(struct ifcvf_hw *hw, u64 offset, 267 const void *src, int length) 268 { 269 const u8 *p; 270 int i; 271 272 p = src; 273 WARN_ON(offset + length > hw->config_size); 274 for (i = 0; i < length; i++) 275 vp_iowrite8(*p++, hw->dev_cfg + offset + i); 276 } 277 278 static void ifcvf_set_features(struct ifcvf_hw *hw, u64 features) 279 { 280 struct virtio_pci_common_cfg __iomem *cfg = hw->common_cfg; 281 282 vp_iowrite32(0, &cfg->guest_feature_select); 283 vp_iowrite32((u32)features, &cfg->guest_feature); 284 285 vp_iowrite32(1, &cfg->guest_feature_select); 286 vp_iowrite32(features >> 32, &cfg->guest_feature); 287 } 288 289 static int ifcvf_config_features(struct ifcvf_hw *hw) 290 { 291 ifcvf_set_features(hw, hw->req_features); 292 ifcvf_add_status(hw, VIRTIO_CONFIG_S_FEATURES_OK); 293 294 if (!(ifcvf_get_status(hw) & VIRTIO_CONFIG_S_FEATURES_OK)) { 295 IFCVF_ERR(hw->pdev, "Failed to set FEATURES_OK status\n"); 296 return -EIO; 297 } 298 299 return 0; 300 } 301 302 u16 ifcvf_get_vq_state(struct ifcvf_hw *hw, u16 qid) 303 { 304 struct ifcvf_lm_cfg __iomem *ifcvf_lm; 305 void __iomem *avail_idx_addr; 306 u16 last_avail_idx; 307 u32 q_pair_id; 308 309 ifcvf_lm = (struct ifcvf_lm_cfg __iomem *)hw->lm_cfg; 310 q_pair_id = qid / 2; 311 avail_idx_addr = &ifcvf_lm->vring_lm_cfg[q_pair_id].idx_addr[qid % 2]; 312 last_avail_idx = vp_ioread16(avail_idx_addr); 313 314 return last_avail_idx; 315 } 316 317 int ifcvf_set_vq_state(struct ifcvf_hw *hw, u16 qid, u16 num) 318 { 319 struct ifcvf_lm_cfg __iomem *ifcvf_lm; 320 void __iomem *avail_idx_addr; 321 u32 q_pair_id; 322 323 ifcvf_lm = (struct ifcvf_lm_cfg __iomem *)hw->lm_cfg; 324 q_pair_id = qid / 2; 325 avail_idx_addr = &ifcvf_lm->vring_lm_cfg[q_pair_id].idx_addr[qid % 2]; 326 hw->vring[qid].last_avail_idx = num; 327 vp_iowrite16(num, avail_idx_addr); 328 329 return 0; 330 } 331 332 static int ifcvf_hw_enable(struct ifcvf_hw *hw) 333 { 334 struct virtio_pci_common_cfg __iomem *cfg; 335 u32 i; 336 337 cfg = hw->common_cfg; 338 for (i = 0; i < hw->nr_vring; i++) { 339 if (!hw->vring[i].ready) 340 break; 341 342 vp_iowrite16(i, &cfg->queue_select); 343 vp_iowrite64_twopart(hw->vring[i].desc, &cfg->queue_desc_lo, 344 &cfg->queue_desc_hi); 345 vp_iowrite64_twopart(hw->vring[i].avail, &cfg->queue_avail_lo, 346 &cfg->queue_avail_hi); 347 vp_iowrite64_twopart(hw->vring[i].used, &cfg->queue_used_lo, 348 &cfg->queue_used_hi); 349 vp_iowrite16(hw->vring[i].size, &cfg->queue_size); 350 ifcvf_set_vq_state(hw, i, hw->vring[i].last_avail_idx); 351 vp_iowrite16(1, &cfg->queue_enable); 352 } 353 354 return 0; 355 } 356 357 static void ifcvf_hw_disable(struct ifcvf_hw *hw) 358 { 359 u32 i; 360 361 ifcvf_set_config_vector(hw, VIRTIO_MSI_NO_VECTOR); 362 for (i = 0; i < hw->nr_vring; i++) { 363 ifcvf_set_vq_vector(hw, i, VIRTIO_MSI_NO_VECTOR); 364 } 365 } 366 367 int ifcvf_start_hw(struct ifcvf_hw *hw) 368 { 369 ifcvf_reset(hw); 370 ifcvf_add_status(hw, VIRTIO_CONFIG_S_ACKNOWLEDGE); 371 ifcvf_add_status(hw, VIRTIO_CONFIG_S_DRIVER); 372 373 if (ifcvf_config_features(hw) < 0) 374 return -EINVAL; 375 376 if (ifcvf_hw_enable(hw) < 0) 377 return -EINVAL; 378 379 ifcvf_add_status(hw, VIRTIO_CONFIG_S_DRIVER_OK); 380 381 return 0; 382 } 383 384 void ifcvf_stop_hw(struct ifcvf_hw *hw) 385 { 386 ifcvf_hw_disable(hw); 387 ifcvf_reset(hw); 388 } 389 390 void ifcvf_notify_queue(struct ifcvf_hw *hw, u16 qid) 391 { 392 vp_iowrite16(qid, hw->vring[qid].notify_addr); 393 } 394