1 // SPDX-License-Identifier: GPL-2.0 2 3 /* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved. 4 * Copyright (C) 2019-2021 Linaro Ltd. 5 */ 6 7 #include <linux/types.h> 8 #include <linux/device.h> 9 #include <linux/slab.h> 10 #include <linux/bitfield.h> 11 #include <linux/if_rmnet.h> 12 #include <linux/dma-direction.h> 13 14 #include "gsi.h" 15 #include "gsi_trans.h" 16 #include "ipa.h" 17 #include "ipa_data.h" 18 #include "ipa_endpoint.h" 19 #include "ipa_cmd.h" 20 #include "ipa_mem.h" 21 #include "ipa_modem.h" 22 #include "ipa_table.h" 23 #include "ipa_gsi.h" 24 #include "ipa_power.h" 25 26 #define atomic_dec_not_zero(v) atomic_add_unless((v), -1, 0) 27 28 /* Hardware is told about receive buffers once a "batch" has been queued */ 29 #define IPA_REPLENISH_BATCH 16 /* Must be non-zero */ 30 31 /* The amount of RX buffer space consumed by standard skb overhead */ 32 #define IPA_RX_BUFFER_OVERHEAD (PAGE_SIZE - SKB_MAX_ORDER(NET_SKB_PAD, 0)) 33 34 /* Where to find the QMAP mux_id for a packet within modem-supplied metadata */ 35 #define IPA_ENDPOINT_QMAP_METADATA_MASK 0x000000ff /* host byte order */ 36 37 #define IPA_ENDPOINT_RESET_AGGR_RETRY_MAX 3 38 #define IPA_AGGR_TIME_LIMIT 500 /* microseconds */ 39 40 /** enum ipa_status_opcode - status element opcode hardware values */ 41 enum ipa_status_opcode { 42 IPA_STATUS_OPCODE_PACKET = 0x01, 43 IPA_STATUS_OPCODE_DROPPED_PACKET = 0x04, 44 IPA_STATUS_OPCODE_SUSPENDED_PACKET = 0x08, 45 IPA_STATUS_OPCODE_PACKET_2ND_PASS = 0x40, 46 }; 47 48 /** enum ipa_status_exception - status element exception type */ 49 enum ipa_status_exception { 50 /* 0 means no exception */ 51 IPA_STATUS_EXCEPTION_DEAGGR = 0x01, 52 }; 53 54 /* Status element provided by hardware */ 55 struct ipa_status { 56 u8 opcode; /* enum ipa_status_opcode */ 57 u8 exception; /* enum ipa_status_exception */ 58 __le16 mask; 59 __le16 pkt_len; 60 u8 endp_src_idx; 61 u8 endp_dst_idx; 62 __le32 metadata; 63 __le32 flags1; 64 __le64 flags2; 65 __le32 flags3; 66 __le32 flags4; 67 }; 68 69 /* Field masks for struct ipa_status structure fields */ 70 #define IPA_STATUS_MASK_TAG_VALID_FMASK GENMASK(4, 4) 71 #define IPA_STATUS_SRC_IDX_FMASK GENMASK(4, 0) 72 #define IPA_STATUS_DST_IDX_FMASK GENMASK(4, 0) 73 #define IPA_STATUS_FLAGS1_RT_RULE_ID_FMASK GENMASK(31, 22) 74 #define IPA_STATUS_FLAGS2_TAG_FMASK GENMASK_ULL(63, 16) 75 76 static u32 aggr_byte_limit_max(enum ipa_version version) 77 { 78 if (version < IPA_VERSION_4_5) 79 return field_max(aggr_byte_limit_fmask(true)); 80 81 return field_max(aggr_byte_limit_fmask(false)); 82 } 83 84 static bool ipa_endpoint_data_valid_one(struct ipa *ipa, u32 count, 85 const struct ipa_gsi_endpoint_data *all_data, 86 const struct ipa_gsi_endpoint_data *data) 87 { 88 const struct ipa_gsi_endpoint_data *other_data; 89 struct device *dev = &ipa->pdev->dev; 90 enum ipa_endpoint_name other_name; 91 92 if (ipa_gsi_endpoint_data_empty(data)) 93 return true; 94 95 if (!data->toward_ipa) { 96 u32 buffer_size; 97 u32 limit; 98 99 if (data->endpoint.filter_support) { 100 dev_err(dev, "filtering not supported for " 101 "RX endpoint %u\n", 102 data->endpoint_id); 103 return false; 104 } 105 106 /* Nothing more to check for non-AP RX */ 107 if (data->ee_id != GSI_EE_AP) 108 return true; 109 110 buffer_size = data->endpoint.config.rx.buffer_size; 111 /* The buffer size must hold an MTU plus overhead */ 112 limit = IPA_MTU + IPA_RX_BUFFER_OVERHEAD; 113 if (buffer_size < limit) { 114 dev_err(dev, "RX buffer size too small for RX endpoint %u (%u < %u)\n", 115 data->endpoint_id, buffer_size, limit); 116 return false; 117 } 118 119 /* For an endpoint supporting receive aggregation, the 120 * aggregation byte limit defines the point at which an 121 * aggregation window will close. It is programmed into the 122 * IPA hardware as a number of KB. We don't use "hard byte 123 * limit" aggregation, so we need to supply enough space in 124 * a receive buffer to hold a complete MTU plus normal skb 125 * overhead *after* that aggregation byte limit has been 126 * crossed. 127 * 128 * This check just ensures the receive buffer size doesn't 129 * exceed what's representable in the aggregation limit field. 130 */ 131 if (data->endpoint.config.aggregation) { 132 limit += SZ_1K * aggr_byte_limit_max(ipa->version); 133 if (buffer_size - NET_SKB_PAD > limit) { 134 dev_err(dev, "RX buffer size too large for aggregated RX endpoint %u (%u > %u)\n", 135 data->endpoint_id, 136 buffer_size - NET_SKB_PAD, limit); 137 138 return false; 139 } 140 } 141 142 return true; /* Nothing more to check for RX */ 143 } 144 145 if (data->endpoint.config.status_enable) { 146 other_name = data->endpoint.config.tx.status_endpoint; 147 if (other_name >= count) { 148 dev_err(dev, "status endpoint name %u out of range " 149 "for endpoint %u\n", 150 other_name, data->endpoint_id); 151 return false; 152 } 153 154 /* Status endpoint must be defined... */ 155 other_data = &all_data[other_name]; 156 if (ipa_gsi_endpoint_data_empty(other_data)) { 157 dev_err(dev, "DMA endpoint name %u undefined " 158 "for endpoint %u\n", 159 other_name, data->endpoint_id); 160 return false; 161 } 162 163 /* ...and has to be an RX endpoint... */ 164 if (other_data->toward_ipa) { 165 dev_err(dev, 166 "status endpoint for endpoint %u not RX\n", 167 data->endpoint_id); 168 return false; 169 } 170 171 /* ...and if it's to be an AP endpoint... */ 172 if (other_data->ee_id == GSI_EE_AP) { 173 /* ...make sure it has status enabled. */ 174 if (!other_data->endpoint.config.status_enable) { 175 dev_err(dev, 176 "status not enabled for endpoint %u\n", 177 other_data->endpoint_id); 178 return false; 179 } 180 } 181 } 182 183 if (data->endpoint.config.dma_mode) { 184 other_name = data->endpoint.config.dma_endpoint; 185 if (other_name >= count) { 186 dev_err(dev, "DMA endpoint name %u out of range " 187 "for endpoint %u\n", 188 other_name, data->endpoint_id); 189 return false; 190 } 191 192 other_data = &all_data[other_name]; 193 if (ipa_gsi_endpoint_data_empty(other_data)) { 194 dev_err(dev, "DMA endpoint name %u undefined " 195 "for endpoint %u\n", 196 other_name, data->endpoint_id); 197 return false; 198 } 199 } 200 201 return true; 202 } 203 204 static bool ipa_endpoint_data_valid(struct ipa *ipa, u32 count, 205 const struct ipa_gsi_endpoint_data *data) 206 { 207 const struct ipa_gsi_endpoint_data *dp = data; 208 struct device *dev = &ipa->pdev->dev; 209 enum ipa_endpoint_name name; 210 211 if (count > IPA_ENDPOINT_COUNT) { 212 dev_err(dev, "too many endpoints specified (%u > %u)\n", 213 count, IPA_ENDPOINT_COUNT); 214 return false; 215 } 216 217 /* Make sure needed endpoints have defined data */ 218 if (ipa_gsi_endpoint_data_empty(&data[IPA_ENDPOINT_AP_COMMAND_TX])) { 219 dev_err(dev, "command TX endpoint not defined\n"); 220 return false; 221 } 222 if (ipa_gsi_endpoint_data_empty(&data[IPA_ENDPOINT_AP_LAN_RX])) { 223 dev_err(dev, "LAN RX endpoint not defined\n"); 224 return false; 225 } 226 if (ipa_gsi_endpoint_data_empty(&data[IPA_ENDPOINT_AP_MODEM_TX])) { 227 dev_err(dev, "AP->modem TX endpoint not defined\n"); 228 return false; 229 } 230 if (ipa_gsi_endpoint_data_empty(&data[IPA_ENDPOINT_AP_MODEM_RX])) { 231 dev_err(dev, "AP<-modem RX endpoint not defined\n"); 232 return false; 233 } 234 235 for (name = 0; name < count; name++, dp++) 236 if (!ipa_endpoint_data_valid_one(ipa, count, data, dp)) 237 return false; 238 239 return true; 240 } 241 242 /* Allocate a transaction to use on a non-command endpoint */ 243 static struct gsi_trans *ipa_endpoint_trans_alloc(struct ipa_endpoint *endpoint, 244 u32 tre_count) 245 { 246 struct gsi *gsi = &endpoint->ipa->gsi; 247 u32 channel_id = endpoint->channel_id; 248 enum dma_data_direction direction; 249 250 direction = endpoint->toward_ipa ? DMA_TO_DEVICE : DMA_FROM_DEVICE; 251 252 return gsi_channel_trans_alloc(gsi, channel_id, tre_count, direction); 253 } 254 255 /* suspend_delay represents suspend for RX, delay for TX endpoints. 256 * Note that suspend is not supported starting with IPA v4.0, and 257 * delay mode should not be used starting with IPA v4.2. 258 */ 259 static bool 260 ipa_endpoint_init_ctrl(struct ipa_endpoint *endpoint, bool suspend_delay) 261 { 262 u32 offset = IPA_REG_ENDP_INIT_CTRL_N_OFFSET(endpoint->endpoint_id); 263 struct ipa *ipa = endpoint->ipa; 264 bool state; 265 u32 mask; 266 u32 val; 267 268 if (endpoint->toward_ipa) 269 WARN_ON(ipa->version >= IPA_VERSION_4_2); 270 else 271 WARN_ON(ipa->version >= IPA_VERSION_4_0); 272 273 mask = endpoint->toward_ipa ? ENDP_DELAY_FMASK : ENDP_SUSPEND_FMASK; 274 275 val = ioread32(ipa->reg_virt + offset); 276 state = !!(val & mask); 277 278 /* Don't bother if it's already in the requested state */ 279 if (suspend_delay != state) { 280 val ^= mask; 281 iowrite32(val, ipa->reg_virt + offset); 282 } 283 284 return state; 285 } 286 287 /* We don't care what the previous state was for delay mode */ 288 static void 289 ipa_endpoint_program_delay(struct ipa_endpoint *endpoint, bool enable) 290 { 291 /* Delay mode should not be used for IPA v4.2+ */ 292 WARN_ON(endpoint->ipa->version >= IPA_VERSION_4_2); 293 WARN_ON(!endpoint->toward_ipa); 294 295 (void)ipa_endpoint_init_ctrl(endpoint, enable); 296 } 297 298 static bool ipa_endpoint_aggr_active(struct ipa_endpoint *endpoint) 299 { 300 u32 mask = BIT(endpoint->endpoint_id); 301 struct ipa *ipa = endpoint->ipa; 302 u32 offset; 303 u32 val; 304 305 WARN_ON(!(mask & ipa->available)); 306 307 offset = ipa_reg_state_aggr_active_offset(ipa->version); 308 val = ioread32(ipa->reg_virt + offset); 309 310 return !!(val & mask); 311 } 312 313 static void ipa_endpoint_force_close(struct ipa_endpoint *endpoint) 314 { 315 u32 mask = BIT(endpoint->endpoint_id); 316 struct ipa *ipa = endpoint->ipa; 317 318 WARN_ON(!(mask & ipa->available)); 319 320 iowrite32(mask, ipa->reg_virt + IPA_REG_AGGR_FORCE_CLOSE_OFFSET); 321 } 322 323 /** 324 * ipa_endpoint_suspend_aggr() - Emulate suspend interrupt 325 * @endpoint: Endpoint on which to emulate a suspend 326 * 327 * Emulate suspend IPA interrupt to unsuspend an endpoint suspended 328 * with an open aggregation frame. This is to work around a hardware 329 * issue in IPA version 3.5.1 where the suspend interrupt will not be 330 * generated when it should be. 331 */ 332 static void ipa_endpoint_suspend_aggr(struct ipa_endpoint *endpoint) 333 { 334 struct ipa *ipa = endpoint->ipa; 335 336 if (!endpoint->data->aggregation) 337 return; 338 339 /* Nothing to do if the endpoint doesn't have aggregation open */ 340 if (!ipa_endpoint_aggr_active(endpoint)) 341 return; 342 343 /* Force close aggregation */ 344 ipa_endpoint_force_close(endpoint); 345 346 ipa_interrupt_simulate_suspend(ipa->interrupt); 347 } 348 349 /* Returns previous suspend state (true means suspend was enabled) */ 350 static bool 351 ipa_endpoint_program_suspend(struct ipa_endpoint *endpoint, bool enable) 352 { 353 bool suspended; 354 355 if (endpoint->ipa->version >= IPA_VERSION_4_0) 356 return enable; /* For IPA v4.0+, no change made */ 357 358 WARN_ON(endpoint->toward_ipa); 359 360 suspended = ipa_endpoint_init_ctrl(endpoint, enable); 361 362 /* A client suspended with an open aggregation frame will not 363 * generate a SUSPEND IPA interrupt. If enabling suspend, have 364 * ipa_endpoint_suspend_aggr() handle this. 365 */ 366 if (enable && !suspended) 367 ipa_endpoint_suspend_aggr(endpoint); 368 369 return suspended; 370 } 371 372 /* Put all modem RX endpoints into suspend mode, and stop transmission 373 * on all modem TX endpoints. Prior to IPA v4.2, endpoint DELAY mode is 374 * used for TX endpoints; starting with IPA v4.2 we use GSI channel flow 375 * control instead. 376 */ 377 void ipa_endpoint_modem_pause_all(struct ipa *ipa, bool enable) 378 { 379 u32 endpoint_id; 380 381 for (endpoint_id = 0; endpoint_id < IPA_ENDPOINT_MAX; endpoint_id++) { 382 struct ipa_endpoint *endpoint = &ipa->endpoint[endpoint_id]; 383 384 if (endpoint->ee_id != GSI_EE_MODEM) 385 continue; 386 387 if (!endpoint->toward_ipa) 388 (void)ipa_endpoint_program_suspend(endpoint, enable); 389 else if (ipa->version < IPA_VERSION_4_2) 390 ipa_endpoint_program_delay(endpoint, enable); 391 else 392 gsi_modem_channel_flow_control(&ipa->gsi, 393 endpoint->channel_id, 394 enable); 395 } 396 } 397 398 /* Reset all modem endpoints to use the default exception endpoint */ 399 int ipa_endpoint_modem_exception_reset_all(struct ipa *ipa) 400 { 401 u32 initialized = ipa->initialized; 402 struct gsi_trans *trans; 403 u32 count; 404 405 /* We need one command per modem TX endpoint. We can get an upper 406 * bound on that by assuming all initialized endpoints are modem->IPA. 407 * That won't happen, and we could be more precise, but this is fine 408 * for now. End the transaction with commands to clear the pipeline. 409 */ 410 count = hweight32(initialized) + ipa_cmd_pipeline_clear_count(); 411 trans = ipa_cmd_trans_alloc(ipa, count); 412 if (!trans) { 413 dev_err(&ipa->pdev->dev, 414 "no transaction to reset modem exception endpoints\n"); 415 return -EBUSY; 416 } 417 418 while (initialized) { 419 u32 endpoint_id = __ffs(initialized); 420 struct ipa_endpoint *endpoint; 421 u32 offset; 422 423 initialized ^= BIT(endpoint_id); 424 425 /* We only reset modem TX endpoints */ 426 endpoint = &ipa->endpoint[endpoint_id]; 427 if (!(endpoint->ee_id == GSI_EE_MODEM && endpoint->toward_ipa)) 428 continue; 429 430 offset = IPA_REG_ENDP_STATUS_N_OFFSET(endpoint_id); 431 432 /* Value written is 0, and all bits are updated. That 433 * means status is disabled on the endpoint, and as a 434 * result all other fields in the register are ignored. 435 */ 436 ipa_cmd_register_write_add(trans, offset, 0, ~0, false); 437 } 438 439 ipa_cmd_pipeline_clear_add(trans); 440 441 /* XXX This should have a 1 second timeout */ 442 gsi_trans_commit_wait(trans); 443 444 ipa_cmd_pipeline_clear_wait(ipa); 445 446 return 0; 447 } 448 449 static void ipa_endpoint_init_cfg(struct ipa_endpoint *endpoint) 450 { 451 u32 offset = IPA_REG_ENDP_INIT_CFG_N_OFFSET(endpoint->endpoint_id); 452 enum ipa_cs_offload_en enabled; 453 u32 val = 0; 454 455 /* FRAG_OFFLOAD_EN is 0 */ 456 if (endpoint->data->checksum) { 457 enum ipa_version version = endpoint->ipa->version; 458 459 if (endpoint->toward_ipa) { 460 u32 checksum_offset; 461 462 /* Checksum header offset is in 4-byte units */ 463 checksum_offset = sizeof(struct rmnet_map_header); 464 checksum_offset /= sizeof(u32); 465 val |= u32_encode_bits(checksum_offset, 466 CS_METADATA_HDR_OFFSET_FMASK); 467 468 enabled = version < IPA_VERSION_4_5 469 ? IPA_CS_OFFLOAD_UL 470 : IPA_CS_OFFLOAD_INLINE; 471 } else { 472 enabled = version < IPA_VERSION_4_5 473 ? IPA_CS_OFFLOAD_DL 474 : IPA_CS_OFFLOAD_INLINE; 475 } 476 } else { 477 enabled = IPA_CS_OFFLOAD_NONE; 478 } 479 val |= u32_encode_bits(enabled, CS_OFFLOAD_EN_FMASK); 480 /* CS_GEN_QMB_MASTER_SEL is 0 */ 481 482 iowrite32(val, endpoint->ipa->reg_virt + offset); 483 } 484 485 static void ipa_endpoint_init_nat(struct ipa_endpoint *endpoint) 486 { 487 u32 offset; 488 u32 val; 489 490 if (!endpoint->toward_ipa) 491 return; 492 493 offset = IPA_REG_ENDP_INIT_NAT_N_OFFSET(endpoint->endpoint_id); 494 val = u32_encode_bits(IPA_NAT_BYPASS, NAT_EN_FMASK); 495 496 iowrite32(val, endpoint->ipa->reg_virt + offset); 497 } 498 499 static u32 500 ipa_qmap_header_size(enum ipa_version version, struct ipa_endpoint *endpoint) 501 { 502 u32 header_size = sizeof(struct rmnet_map_header); 503 504 /* Without checksum offload, we just have the MAP header */ 505 if (!endpoint->data->checksum) 506 return header_size; 507 508 if (version < IPA_VERSION_4_5) { 509 /* Checksum header inserted for AP TX endpoints only */ 510 if (endpoint->toward_ipa) 511 header_size += sizeof(struct rmnet_map_ul_csum_header); 512 } else { 513 /* Checksum header is used in both directions */ 514 header_size += sizeof(struct rmnet_map_v5_csum_header); 515 } 516 517 return header_size; 518 } 519 520 /** 521 * ipa_endpoint_init_hdr() - Initialize HDR endpoint configuration register 522 * @endpoint: Endpoint pointer 523 * 524 * We program QMAP endpoints so each packet received is preceded by a QMAP 525 * header structure. The QMAP header contains a 1-byte mux_id and 2-byte 526 * packet size field, and we have the IPA hardware populate both for each 527 * received packet. The header is configured (in the HDR_EXT register) 528 * to use big endian format. 529 * 530 * The packet size is written into the QMAP header's pkt_len field. That 531 * location is defined here using the HDR_OFST_PKT_SIZE field. 532 * 533 * The mux_id comes from a 4-byte metadata value supplied with each packet 534 * by the modem. It is *not* a QMAP header, but it does contain the mux_id 535 * value that we want, in its low-order byte. A bitmask defined in the 536 * endpoint's METADATA_MASK register defines which byte within the modem 537 * metadata contains the mux_id. And the OFST_METADATA field programmed 538 * here indicates where the extracted byte should be placed within the QMAP 539 * header. 540 */ 541 static void ipa_endpoint_init_hdr(struct ipa_endpoint *endpoint) 542 { 543 u32 offset = IPA_REG_ENDP_INIT_HDR_N_OFFSET(endpoint->endpoint_id); 544 struct ipa *ipa = endpoint->ipa; 545 u32 val = 0; 546 547 if (endpoint->data->qmap) { 548 enum ipa_version version = ipa->version; 549 size_t header_size; 550 551 header_size = ipa_qmap_header_size(version, endpoint); 552 val = ipa_header_size_encoded(version, header_size); 553 554 /* Define how to fill fields in a received QMAP header */ 555 if (!endpoint->toward_ipa) { 556 u32 offset; /* Field offset within header */ 557 558 /* Where IPA will write the metadata value */ 559 offset = offsetof(struct rmnet_map_header, mux_id); 560 val |= ipa_metadata_offset_encoded(version, offset); 561 562 /* Where IPA will write the length */ 563 offset = offsetof(struct rmnet_map_header, pkt_len); 564 /* Upper bits are stored in HDR_EXT with IPA v4.5 */ 565 if (version >= IPA_VERSION_4_5) 566 offset &= field_mask(HDR_OFST_PKT_SIZE_FMASK); 567 568 val |= HDR_OFST_PKT_SIZE_VALID_FMASK; 569 val |= u32_encode_bits(offset, HDR_OFST_PKT_SIZE_FMASK); 570 } 571 /* For QMAP TX, metadata offset is 0 (modem assumes this) */ 572 val |= HDR_OFST_METADATA_VALID_FMASK; 573 574 /* HDR_ADDITIONAL_CONST_LEN is 0; (RX only) */ 575 /* HDR_A5_MUX is 0 */ 576 /* HDR_LEN_INC_DEAGG_HDR is 0 */ 577 /* HDR_METADATA_REG_VALID is 0 (TX only, version < v4.5) */ 578 } 579 580 iowrite32(val, ipa->reg_virt + offset); 581 } 582 583 static void ipa_endpoint_init_hdr_ext(struct ipa_endpoint *endpoint) 584 { 585 u32 offset = IPA_REG_ENDP_INIT_HDR_EXT_N_OFFSET(endpoint->endpoint_id); 586 u32 pad_align = endpoint->data->rx.pad_align; 587 struct ipa *ipa = endpoint->ipa; 588 u32 val = 0; 589 590 val |= HDR_ENDIANNESS_FMASK; /* big endian */ 591 592 /* A QMAP header contains a 6 bit pad field at offset 0. The RMNet 593 * driver assumes this field is meaningful in packets it receives, 594 * and assumes the header's payload length includes that padding. 595 * The RMNet driver does *not* pad packets it sends, however, so 596 * the pad field (although 0) should be ignored. 597 */ 598 if (endpoint->data->qmap && !endpoint->toward_ipa) { 599 val |= HDR_TOTAL_LEN_OR_PAD_VALID_FMASK; 600 /* HDR_TOTAL_LEN_OR_PAD is 0 (pad, not total_len) */ 601 val |= HDR_PAYLOAD_LEN_INC_PADDING_FMASK; 602 /* HDR_TOTAL_LEN_OR_PAD_OFFSET is 0 */ 603 } 604 605 /* HDR_PAYLOAD_LEN_INC_PADDING is 0 */ 606 if (!endpoint->toward_ipa) 607 val |= u32_encode_bits(pad_align, HDR_PAD_TO_ALIGNMENT_FMASK); 608 609 /* IPA v4.5 adds some most-significant bits to a few fields, 610 * two of which are defined in the HDR (not HDR_EXT) register. 611 */ 612 if (ipa->version >= IPA_VERSION_4_5) { 613 /* HDR_TOTAL_LEN_OR_PAD_OFFSET is 0, so MSB is 0 */ 614 if (endpoint->data->qmap && !endpoint->toward_ipa) { 615 u32 offset; 616 617 offset = offsetof(struct rmnet_map_header, pkt_len); 618 offset >>= hweight32(HDR_OFST_PKT_SIZE_FMASK); 619 val |= u32_encode_bits(offset, 620 HDR_OFST_PKT_SIZE_MSB_FMASK); 621 /* HDR_ADDITIONAL_CONST_LEN is 0 so MSB is 0 */ 622 } 623 } 624 iowrite32(val, ipa->reg_virt + offset); 625 } 626 627 static void ipa_endpoint_init_hdr_metadata_mask(struct ipa_endpoint *endpoint) 628 { 629 u32 endpoint_id = endpoint->endpoint_id; 630 u32 val = 0; 631 u32 offset; 632 633 if (endpoint->toward_ipa) 634 return; /* Register not valid for TX endpoints */ 635 636 offset = IPA_REG_ENDP_INIT_HDR_METADATA_MASK_N_OFFSET(endpoint_id); 637 638 /* Note that HDR_ENDIANNESS indicates big endian header fields */ 639 if (endpoint->data->qmap) 640 val = (__force u32)cpu_to_be32(IPA_ENDPOINT_QMAP_METADATA_MASK); 641 642 iowrite32(val, endpoint->ipa->reg_virt + offset); 643 } 644 645 static void ipa_endpoint_init_mode(struct ipa_endpoint *endpoint) 646 { 647 u32 offset = IPA_REG_ENDP_INIT_MODE_N_OFFSET(endpoint->endpoint_id); 648 u32 val; 649 650 if (!endpoint->toward_ipa) 651 return; /* Register not valid for RX endpoints */ 652 653 if (endpoint->data->dma_mode) { 654 enum ipa_endpoint_name name = endpoint->data->dma_endpoint; 655 u32 dma_endpoint_id; 656 657 dma_endpoint_id = endpoint->ipa->name_map[name]->endpoint_id; 658 659 val = u32_encode_bits(IPA_DMA, MODE_FMASK); 660 val |= u32_encode_bits(dma_endpoint_id, DEST_PIPE_INDEX_FMASK); 661 } else { 662 val = u32_encode_bits(IPA_BASIC, MODE_FMASK); 663 } 664 /* All other bits unspecified (and 0) */ 665 666 iowrite32(val, endpoint->ipa->reg_virt + offset); 667 } 668 669 /* Compute the aggregation size value to use for a given buffer size */ 670 static u32 ipa_aggr_size_kb(u32 rx_buffer_size) 671 { 672 /* We don't use "hard byte limit" aggregation, so we define the 673 * aggregation limit such that our buffer has enough space *after* 674 * that limit to receive a full MTU of data, plus overhead. 675 */ 676 rx_buffer_size -= IPA_MTU + IPA_RX_BUFFER_OVERHEAD; 677 678 return rx_buffer_size / SZ_1K; 679 } 680 681 /* Encoded values for AGGR endpoint register fields */ 682 static u32 aggr_byte_limit_encoded(enum ipa_version version, u32 limit) 683 { 684 if (version < IPA_VERSION_4_5) 685 return u32_encode_bits(limit, aggr_byte_limit_fmask(true)); 686 687 return u32_encode_bits(limit, aggr_byte_limit_fmask(false)); 688 } 689 690 /* Encode the aggregation timer limit (microseconds) based on IPA version */ 691 static u32 aggr_time_limit_encoded(enum ipa_version version, u32 limit) 692 { 693 u32 gran_sel; 694 u32 fmask; 695 u32 val; 696 697 if (version < IPA_VERSION_4_5) { 698 /* We set aggregation granularity in ipa_hardware_config() */ 699 limit = DIV_ROUND_CLOSEST(limit, IPA_AGGR_GRANULARITY); 700 701 return u32_encode_bits(limit, aggr_time_limit_fmask(true)); 702 } 703 704 /* IPA v4.5 expresses the time limit using Qtime. The AP has 705 * pulse generators 0 and 1 available, which were configured 706 * in ipa_qtime_config() to have granularity 100 usec and 707 * 1 msec, respectively. Use pulse generator 0 if possible, 708 * otherwise fall back to pulse generator 1. 709 */ 710 fmask = aggr_time_limit_fmask(false); 711 val = DIV_ROUND_CLOSEST(limit, 100); 712 if (val > field_max(fmask)) { 713 /* Have to use pulse generator 1 (millisecond granularity) */ 714 gran_sel = AGGR_GRAN_SEL_FMASK; 715 val = DIV_ROUND_CLOSEST(limit, 1000); 716 } else { 717 /* We can use pulse generator 0 (100 usec granularity) */ 718 gran_sel = 0; 719 } 720 721 return gran_sel | u32_encode_bits(val, fmask); 722 } 723 724 static u32 aggr_sw_eof_active_encoded(enum ipa_version version, bool enabled) 725 { 726 u32 val = enabled ? 1 : 0; 727 728 if (version < IPA_VERSION_4_5) 729 return u32_encode_bits(val, aggr_sw_eof_active_fmask(true)); 730 731 return u32_encode_bits(val, aggr_sw_eof_active_fmask(false)); 732 } 733 734 static void ipa_endpoint_init_aggr(struct ipa_endpoint *endpoint) 735 { 736 u32 offset = IPA_REG_ENDP_INIT_AGGR_N_OFFSET(endpoint->endpoint_id); 737 enum ipa_version version = endpoint->ipa->version; 738 u32 val = 0; 739 740 if (endpoint->data->aggregation) { 741 if (!endpoint->toward_ipa) { 742 const struct ipa_endpoint_rx_data *rx_data; 743 u32 buffer_size; 744 bool close_eof; 745 u32 limit; 746 747 rx_data = &endpoint->data->rx; 748 val |= u32_encode_bits(IPA_ENABLE_AGGR, AGGR_EN_FMASK); 749 val |= u32_encode_bits(IPA_GENERIC, AGGR_TYPE_FMASK); 750 751 buffer_size = rx_data->buffer_size; 752 limit = ipa_aggr_size_kb(buffer_size - NET_SKB_PAD); 753 val |= aggr_byte_limit_encoded(version, limit); 754 755 limit = IPA_AGGR_TIME_LIMIT; 756 val |= aggr_time_limit_encoded(version, limit); 757 758 /* AGGR_PKT_LIMIT is 0 (unlimited) */ 759 760 close_eof = rx_data->aggr_close_eof; 761 val |= aggr_sw_eof_active_encoded(version, close_eof); 762 763 /* AGGR_HARD_BYTE_LIMIT_ENABLE is 0 */ 764 } else { 765 val |= u32_encode_bits(IPA_ENABLE_DEAGGR, 766 AGGR_EN_FMASK); 767 val |= u32_encode_bits(IPA_QCMAP, AGGR_TYPE_FMASK); 768 /* other fields ignored */ 769 } 770 /* AGGR_FORCE_CLOSE is 0 */ 771 /* AGGR_GRAN_SEL is 0 for IPA v4.5 */ 772 } else { 773 val |= u32_encode_bits(IPA_BYPASS_AGGR, AGGR_EN_FMASK); 774 /* other fields ignored */ 775 } 776 777 iowrite32(val, endpoint->ipa->reg_virt + offset); 778 } 779 780 /* Return the Qtime-based head-of-line blocking timer value that 781 * represents the given number of microseconds. The result 782 * includes both the timer value and the selected timer granularity. 783 */ 784 static u32 hol_block_timer_qtime_val(struct ipa *ipa, u32 microseconds) 785 { 786 u32 gran_sel; 787 u32 val; 788 789 /* IPA v4.5 expresses time limits using Qtime. The AP has 790 * pulse generators 0 and 1 available, which were configured 791 * in ipa_qtime_config() to have granularity 100 usec and 792 * 1 msec, respectively. Use pulse generator 0 if possible, 793 * otherwise fall back to pulse generator 1. 794 */ 795 val = DIV_ROUND_CLOSEST(microseconds, 100); 796 if (val > field_max(TIME_LIMIT_FMASK)) { 797 /* Have to use pulse generator 1 (millisecond granularity) */ 798 gran_sel = GRAN_SEL_FMASK; 799 val = DIV_ROUND_CLOSEST(microseconds, 1000); 800 } else { 801 /* We can use pulse generator 0 (100 usec granularity) */ 802 gran_sel = 0; 803 } 804 805 return gran_sel | u32_encode_bits(val, TIME_LIMIT_FMASK); 806 } 807 808 /* The head-of-line blocking timer is defined as a tick count. For 809 * IPA version 4.5 the tick count is based on the Qtimer, which is 810 * derived from the 19.2 MHz SoC XO clock. For older IPA versions 811 * each tick represents 128 cycles of the IPA core clock. 812 * 813 * Return the encoded value that should be written to that register 814 * that represents the timeout period provided. For IPA v4.2 this 815 * encodes a base and scale value, while for earlier versions the 816 * value is a simple tick count. 817 */ 818 static u32 hol_block_timer_val(struct ipa *ipa, u32 microseconds) 819 { 820 u32 width; 821 u32 scale; 822 u64 ticks; 823 u64 rate; 824 u32 high; 825 u32 val; 826 827 if (!microseconds) 828 return 0; /* Nothing to compute if timer period is 0 */ 829 830 if (ipa->version >= IPA_VERSION_4_5) 831 return hol_block_timer_qtime_val(ipa, microseconds); 832 833 /* Use 64 bit arithmetic to avoid overflow... */ 834 rate = ipa_core_clock_rate(ipa); 835 ticks = DIV_ROUND_CLOSEST(microseconds * rate, 128 * USEC_PER_SEC); 836 /* ...but we still need to fit into a 32-bit register */ 837 WARN_ON(ticks > U32_MAX); 838 839 /* IPA v3.5.1 through v4.1 just record the tick count */ 840 if (ipa->version < IPA_VERSION_4_2) 841 return (u32)ticks; 842 843 /* For IPA v4.2, the tick count is represented by base and 844 * scale fields within the 32-bit timer register, where: 845 * ticks = base << scale; 846 * The best precision is achieved when the base value is as 847 * large as possible. Find the highest set bit in the tick 848 * count, and extract the number of bits in the base field 849 * such that high bit is included. 850 */ 851 high = fls(ticks); /* 1..32 */ 852 width = HWEIGHT32(BASE_VALUE_FMASK); 853 scale = high > width ? high - width : 0; 854 if (scale) { 855 /* If we're scaling, round up to get a closer result */ 856 ticks += 1 << (scale - 1); 857 /* High bit was set, so rounding might have affected it */ 858 if (fls(ticks) != high) 859 scale++; 860 } 861 862 val = u32_encode_bits(scale, SCALE_FMASK); 863 val |= u32_encode_bits(ticks >> scale, BASE_VALUE_FMASK); 864 865 return val; 866 } 867 868 /* If microseconds is 0, timeout is immediate */ 869 static void ipa_endpoint_init_hol_block_timer(struct ipa_endpoint *endpoint, 870 u32 microseconds) 871 { 872 u32 endpoint_id = endpoint->endpoint_id; 873 struct ipa *ipa = endpoint->ipa; 874 u32 offset; 875 u32 val; 876 877 /* This should only be changed when HOL_BLOCK_EN is disabled */ 878 offset = IPA_REG_ENDP_INIT_HOL_BLOCK_TIMER_N_OFFSET(endpoint_id); 879 val = hol_block_timer_val(ipa, microseconds); 880 iowrite32(val, ipa->reg_virt + offset); 881 } 882 883 static void 884 ipa_endpoint_init_hol_block_en(struct ipa_endpoint *endpoint, bool enable) 885 { 886 u32 endpoint_id = endpoint->endpoint_id; 887 u32 offset; 888 u32 val; 889 890 val = enable ? HOL_BLOCK_EN_FMASK : 0; 891 offset = IPA_REG_ENDP_INIT_HOL_BLOCK_EN_N_OFFSET(endpoint_id); 892 iowrite32(val, endpoint->ipa->reg_virt + offset); 893 /* When enabling, the register must be written twice for IPA v4.5+ */ 894 if (enable && endpoint->ipa->version >= IPA_VERSION_4_5) 895 iowrite32(val, endpoint->ipa->reg_virt + offset); 896 } 897 898 /* Assumes HOL_BLOCK is in disabled state */ 899 static void ipa_endpoint_init_hol_block_enable(struct ipa_endpoint *endpoint, 900 u32 microseconds) 901 { 902 ipa_endpoint_init_hol_block_timer(endpoint, microseconds); 903 ipa_endpoint_init_hol_block_en(endpoint, true); 904 } 905 906 static void ipa_endpoint_init_hol_block_disable(struct ipa_endpoint *endpoint) 907 { 908 ipa_endpoint_init_hol_block_en(endpoint, false); 909 } 910 911 void ipa_endpoint_modem_hol_block_clear_all(struct ipa *ipa) 912 { 913 u32 i; 914 915 for (i = 0; i < IPA_ENDPOINT_MAX; i++) { 916 struct ipa_endpoint *endpoint = &ipa->endpoint[i]; 917 918 if (endpoint->toward_ipa || endpoint->ee_id != GSI_EE_MODEM) 919 continue; 920 921 ipa_endpoint_init_hol_block_disable(endpoint); 922 ipa_endpoint_init_hol_block_enable(endpoint, 0); 923 } 924 } 925 926 static void ipa_endpoint_init_deaggr(struct ipa_endpoint *endpoint) 927 { 928 u32 offset = IPA_REG_ENDP_INIT_DEAGGR_N_OFFSET(endpoint->endpoint_id); 929 u32 val = 0; 930 931 if (!endpoint->toward_ipa) 932 return; /* Register not valid for RX endpoints */ 933 934 /* DEAGGR_HDR_LEN is 0 */ 935 /* PACKET_OFFSET_VALID is 0 */ 936 /* PACKET_OFFSET_LOCATION is ignored (not valid) */ 937 /* MAX_PACKET_LEN is 0 (not enforced) */ 938 939 iowrite32(val, endpoint->ipa->reg_virt + offset); 940 } 941 942 static void ipa_endpoint_init_rsrc_grp(struct ipa_endpoint *endpoint) 943 { 944 u32 offset = IPA_REG_ENDP_INIT_RSRC_GRP_N_OFFSET(endpoint->endpoint_id); 945 struct ipa *ipa = endpoint->ipa; 946 u32 val; 947 948 val = rsrc_grp_encoded(ipa->version, endpoint->data->resource_group); 949 iowrite32(val, ipa->reg_virt + offset); 950 } 951 952 static void ipa_endpoint_init_seq(struct ipa_endpoint *endpoint) 953 { 954 u32 offset = IPA_REG_ENDP_INIT_SEQ_N_OFFSET(endpoint->endpoint_id); 955 u32 val = 0; 956 957 if (!endpoint->toward_ipa) 958 return; /* Register not valid for RX endpoints */ 959 960 /* Low-order byte configures primary packet processing */ 961 val |= u32_encode_bits(endpoint->data->tx.seq_type, SEQ_TYPE_FMASK); 962 963 /* Second byte configures replicated packet processing */ 964 val |= u32_encode_bits(endpoint->data->tx.seq_rep_type, 965 SEQ_REP_TYPE_FMASK); 966 967 iowrite32(val, endpoint->ipa->reg_virt + offset); 968 } 969 970 /** 971 * ipa_endpoint_skb_tx() - Transmit a socket buffer 972 * @endpoint: Endpoint pointer 973 * @skb: Socket buffer to send 974 * 975 * Returns: 0 if successful, or a negative error code 976 */ 977 int ipa_endpoint_skb_tx(struct ipa_endpoint *endpoint, struct sk_buff *skb) 978 { 979 struct gsi_trans *trans; 980 u32 nr_frags; 981 int ret; 982 983 /* Make sure source endpoint's TLV FIFO has enough entries to 984 * hold the linear portion of the skb and all its fragments. 985 * If not, see if we can linearize it before giving up. 986 */ 987 nr_frags = skb_shinfo(skb)->nr_frags; 988 if (1 + nr_frags > endpoint->trans_tre_max) { 989 if (skb_linearize(skb)) 990 return -E2BIG; 991 nr_frags = 0; 992 } 993 994 trans = ipa_endpoint_trans_alloc(endpoint, 1 + nr_frags); 995 if (!trans) 996 return -EBUSY; 997 998 ret = gsi_trans_skb_add(trans, skb); 999 if (ret) 1000 goto err_trans_free; 1001 trans->data = skb; /* transaction owns skb now */ 1002 1003 gsi_trans_commit(trans, !netdev_xmit_more()); 1004 1005 return 0; 1006 1007 err_trans_free: 1008 gsi_trans_free(trans); 1009 1010 return -ENOMEM; 1011 } 1012 1013 static void ipa_endpoint_status(struct ipa_endpoint *endpoint) 1014 { 1015 u32 endpoint_id = endpoint->endpoint_id; 1016 struct ipa *ipa = endpoint->ipa; 1017 u32 val = 0; 1018 u32 offset; 1019 1020 offset = IPA_REG_ENDP_STATUS_N_OFFSET(endpoint_id); 1021 1022 if (endpoint->data->status_enable) { 1023 val |= STATUS_EN_FMASK; 1024 if (endpoint->toward_ipa) { 1025 enum ipa_endpoint_name name; 1026 u32 status_endpoint_id; 1027 1028 name = endpoint->data->tx.status_endpoint; 1029 status_endpoint_id = ipa->name_map[name]->endpoint_id; 1030 1031 val |= u32_encode_bits(status_endpoint_id, 1032 STATUS_ENDP_FMASK); 1033 } 1034 /* STATUS_LOCATION is 0, meaning status element precedes 1035 * packet (not present for IPA v4.5) 1036 */ 1037 /* STATUS_PKT_SUPPRESS_FMASK is 0 (not present for v3.5.1) */ 1038 } 1039 1040 iowrite32(val, ipa->reg_virt + offset); 1041 } 1042 1043 static int ipa_endpoint_replenish_one(struct ipa_endpoint *endpoint, 1044 struct gsi_trans *trans) 1045 { 1046 struct page *page; 1047 u32 buffer_size; 1048 u32 offset; 1049 u32 len; 1050 int ret; 1051 1052 buffer_size = endpoint->data->rx.buffer_size; 1053 page = dev_alloc_pages(get_order(buffer_size)); 1054 if (!page) 1055 return -ENOMEM; 1056 1057 /* Offset the buffer to make space for skb headroom */ 1058 offset = NET_SKB_PAD; 1059 len = buffer_size - offset; 1060 1061 ret = gsi_trans_page_add(trans, page, len, offset); 1062 if (ret) 1063 __free_pages(page, get_order(buffer_size)); 1064 else 1065 trans->data = page; /* transaction owns page now */ 1066 1067 return ret; 1068 } 1069 1070 /** 1071 * ipa_endpoint_replenish() - Replenish endpoint receive buffers 1072 * @endpoint: Endpoint to be replenished 1073 * 1074 * The IPA hardware can hold a fixed number of receive buffers for an RX 1075 * endpoint, based on the number of entries in the underlying channel ring 1076 * buffer. If an endpoint's "backlog" is non-zero, it indicates how many 1077 * more receive buffers can be supplied to the hardware. Replenishing for 1078 * an endpoint can be disabled, in which case buffers are not queued to 1079 * the hardware. 1080 */ 1081 static void ipa_endpoint_replenish(struct ipa_endpoint *endpoint) 1082 { 1083 struct gsi_trans *trans; 1084 1085 if (!test_bit(IPA_REPLENISH_ENABLED, endpoint->replenish_flags)) 1086 return; 1087 1088 /* Skip it if it's already active */ 1089 if (test_and_set_bit(IPA_REPLENISH_ACTIVE, endpoint->replenish_flags)) 1090 return; 1091 1092 while ((trans = ipa_endpoint_trans_alloc(endpoint, 1))) { 1093 bool doorbell; 1094 1095 if (ipa_endpoint_replenish_one(endpoint, trans)) 1096 goto try_again_later; 1097 1098 1099 /* Ring the doorbell if we've got a full batch */ 1100 doorbell = !(++endpoint->replenish_count % IPA_REPLENISH_BATCH); 1101 gsi_trans_commit(trans, doorbell); 1102 } 1103 1104 clear_bit(IPA_REPLENISH_ACTIVE, endpoint->replenish_flags); 1105 1106 return; 1107 1108 try_again_later: 1109 gsi_trans_free(trans); 1110 clear_bit(IPA_REPLENISH_ACTIVE, endpoint->replenish_flags); 1111 1112 /* Whenever a receive buffer transaction completes we'll try to 1113 * replenish again. It's unlikely, but if we fail to supply even 1114 * one buffer, nothing will trigger another replenish attempt. 1115 * If the hardware has no receive buffers queued, schedule work to 1116 * try replenishing again. 1117 */ 1118 if (gsi_channel_trans_idle(&endpoint->ipa->gsi, endpoint->channel_id)) 1119 schedule_delayed_work(&endpoint->replenish_work, 1120 msecs_to_jiffies(1)); 1121 } 1122 1123 static void ipa_endpoint_replenish_enable(struct ipa_endpoint *endpoint) 1124 { 1125 set_bit(IPA_REPLENISH_ENABLED, endpoint->replenish_flags); 1126 1127 /* Start replenishing if hardware currently has no buffers */ 1128 if (gsi_channel_trans_idle(&endpoint->ipa->gsi, endpoint->channel_id)) 1129 ipa_endpoint_replenish(endpoint); 1130 } 1131 1132 static void ipa_endpoint_replenish_disable(struct ipa_endpoint *endpoint) 1133 { 1134 clear_bit(IPA_REPLENISH_ENABLED, endpoint->replenish_flags); 1135 } 1136 1137 static void ipa_endpoint_replenish_work(struct work_struct *work) 1138 { 1139 struct delayed_work *dwork = to_delayed_work(work); 1140 struct ipa_endpoint *endpoint; 1141 1142 endpoint = container_of(dwork, struct ipa_endpoint, replenish_work); 1143 1144 ipa_endpoint_replenish(endpoint); 1145 } 1146 1147 static void ipa_endpoint_skb_copy(struct ipa_endpoint *endpoint, 1148 void *data, u32 len, u32 extra) 1149 { 1150 struct sk_buff *skb; 1151 1152 if (!endpoint->netdev) 1153 return; 1154 1155 skb = __dev_alloc_skb(len, GFP_ATOMIC); 1156 if (!skb) 1157 return; 1158 1159 /* Copy the data into the socket buffer and receive it */ 1160 skb_put(skb, len); 1161 memcpy(skb->data, data, len); 1162 skb->truesize += extra; 1163 1164 ipa_modem_skb_rx(endpoint->netdev, skb); 1165 } 1166 1167 static bool ipa_endpoint_skb_build(struct ipa_endpoint *endpoint, 1168 struct page *page, u32 len) 1169 { 1170 u32 buffer_size = endpoint->data->rx.buffer_size; 1171 struct sk_buff *skb; 1172 1173 /* Nothing to do if there's no netdev */ 1174 if (!endpoint->netdev) 1175 return false; 1176 1177 WARN_ON(len > SKB_WITH_OVERHEAD(buffer_size - NET_SKB_PAD)); 1178 1179 skb = build_skb(page_address(page), buffer_size); 1180 if (skb) { 1181 /* Reserve the headroom and account for the data */ 1182 skb_reserve(skb, NET_SKB_PAD); 1183 skb_put(skb, len); 1184 } 1185 1186 /* Receive the buffer (or record drop if unable to build it) */ 1187 ipa_modem_skb_rx(endpoint->netdev, skb); 1188 1189 return skb != NULL; 1190 } 1191 1192 /* The format of a packet status element is the same for several status 1193 * types (opcodes). Other types aren't currently supported. 1194 */ 1195 static bool ipa_status_format_packet(enum ipa_status_opcode opcode) 1196 { 1197 switch (opcode) { 1198 case IPA_STATUS_OPCODE_PACKET: 1199 case IPA_STATUS_OPCODE_DROPPED_PACKET: 1200 case IPA_STATUS_OPCODE_SUSPENDED_PACKET: 1201 case IPA_STATUS_OPCODE_PACKET_2ND_PASS: 1202 return true; 1203 default: 1204 return false; 1205 } 1206 } 1207 1208 static bool ipa_endpoint_status_skip(struct ipa_endpoint *endpoint, 1209 const struct ipa_status *status) 1210 { 1211 u32 endpoint_id; 1212 1213 if (!ipa_status_format_packet(status->opcode)) 1214 return true; 1215 if (!status->pkt_len) 1216 return true; 1217 endpoint_id = u8_get_bits(status->endp_dst_idx, 1218 IPA_STATUS_DST_IDX_FMASK); 1219 if (endpoint_id != endpoint->endpoint_id) 1220 return true; 1221 1222 return false; /* Don't skip this packet, process it */ 1223 } 1224 1225 static bool ipa_endpoint_status_tag(struct ipa_endpoint *endpoint, 1226 const struct ipa_status *status) 1227 { 1228 struct ipa_endpoint *command_endpoint; 1229 struct ipa *ipa = endpoint->ipa; 1230 u32 endpoint_id; 1231 1232 if (!le16_get_bits(status->mask, IPA_STATUS_MASK_TAG_VALID_FMASK)) 1233 return false; /* No valid tag */ 1234 1235 /* The status contains a valid tag. We know the packet was sent to 1236 * this endpoint (already verified by ipa_endpoint_status_skip()). 1237 * If the packet came from the AP->command TX endpoint we know 1238 * this packet was sent as part of the pipeline clear process. 1239 */ 1240 endpoint_id = u8_get_bits(status->endp_src_idx, 1241 IPA_STATUS_SRC_IDX_FMASK); 1242 command_endpoint = ipa->name_map[IPA_ENDPOINT_AP_COMMAND_TX]; 1243 if (endpoint_id == command_endpoint->endpoint_id) { 1244 complete(&ipa->completion); 1245 } else { 1246 dev_err(&ipa->pdev->dev, 1247 "unexpected tagged packet from endpoint %u\n", 1248 endpoint_id); 1249 } 1250 1251 return true; 1252 } 1253 1254 /* Return whether the status indicates the packet should be dropped */ 1255 static bool ipa_endpoint_status_drop(struct ipa_endpoint *endpoint, 1256 const struct ipa_status *status) 1257 { 1258 u32 val; 1259 1260 /* If the status indicates a tagged transfer, we'll drop the packet */ 1261 if (ipa_endpoint_status_tag(endpoint, status)) 1262 return true; 1263 1264 /* Deaggregation exceptions we drop; all other types we consume */ 1265 if (status->exception) 1266 return status->exception == IPA_STATUS_EXCEPTION_DEAGGR; 1267 1268 /* Drop the packet if it fails to match a routing rule; otherwise no */ 1269 val = le32_get_bits(status->flags1, IPA_STATUS_FLAGS1_RT_RULE_ID_FMASK); 1270 1271 return val == field_max(IPA_STATUS_FLAGS1_RT_RULE_ID_FMASK); 1272 } 1273 1274 static void ipa_endpoint_status_parse(struct ipa_endpoint *endpoint, 1275 struct page *page, u32 total_len) 1276 { 1277 u32 buffer_size = endpoint->data->rx.buffer_size; 1278 void *data = page_address(page) + NET_SKB_PAD; 1279 u32 unused = buffer_size - total_len; 1280 u32 resid = total_len; 1281 1282 while (resid) { 1283 const struct ipa_status *status = data; 1284 u32 align; 1285 u32 len; 1286 1287 if (resid < sizeof(*status)) { 1288 dev_err(&endpoint->ipa->pdev->dev, 1289 "short message (%u bytes < %zu byte status)\n", 1290 resid, sizeof(*status)); 1291 break; 1292 } 1293 1294 /* Skip over status packets that lack packet data */ 1295 if (ipa_endpoint_status_skip(endpoint, status)) { 1296 data += sizeof(*status); 1297 resid -= sizeof(*status); 1298 continue; 1299 } 1300 1301 /* Compute the amount of buffer space consumed by the packet, 1302 * including the status element. If the hardware is configured 1303 * to pad packet data to an aligned boundary, account for that. 1304 * And if checksum offload is enabled a trailer containing 1305 * computed checksum information will be appended. 1306 */ 1307 align = endpoint->data->rx.pad_align ? : 1; 1308 len = le16_to_cpu(status->pkt_len); 1309 len = sizeof(*status) + ALIGN(len, align); 1310 if (endpoint->data->checksum) 1311 len += sizeof(struct rmnet_map_dl_csum_trailer); 1312 1313 if (!ipa_endpoint_status_drop(endpoint, status)) { 1314 void *data2; 1315 u32 extra; 1316 u32 len2; 1317 1318 /* Client receives only packet data (no status) */ 1319 data2 = data + sizeof(*status); 1320 len2 = le16_to_cpu(status->pkt_len); 1321 1322 /* Have the true size reflect the extra unused space in 1323 * the original receive buffer. Distribute the "cost" 1324 * proportionately across all aggregated packets in the 1325 * buffer. 1326 */ 1327 extra = DIV_ROUND_CLOSEST(unused * len, total_len); 1328 ipa_endpoint_skb_copy(endpoint, data2, len2, extra); 1329 } 1330 1331 /* Consume status and the full packet it describes */ 1332 data += len; 1333 resid -= len; 1334 } 1335 } 1336 1337 /* Complete a TX transaction, command or from ipa_endpoint_skb_tx() */ 1338 static void ipa_endpoint_tx_complete(struct ipa_endpoint *endpoint, 1339 struct gsi_trans *trans) 1340 { 1341 } 1342 1343 /* Complete transaction initiated in ipa_endpoint_replenish_one() */ 1344 static void ipa_endpoint_rx_complete(struct ipa_endpoint *endpoint, 1345 struct gsi_trans *trans) 1346 { 1347 struct page *page; 1348 1349 if (trans->cancelled) 1350 goto done; 1351 1352 /* Parse or build a socket buffer using the actual received length */ 1353 page = trans->data; 1354 if (endpoint->data->status_enable) 1355 ipa_endpoint_status_parse(endpoint, page, trans->len); 1356 else if (ipa_endpoint_skb_build(endpoint, page, trans->len)) 1357 trans->data = NULL; /* Pages have been consumed */ 1358 done: 1359 ipa_endpoint_replenish(endpoint); 1360 } 1361 1362 void ipa_endpoint_trans_complete(struct ipa_endpoint *endpoint, 1363 struct gsi_trans *trans) 1364 { 1365 if (endpoint->toward_ipa) 1366 ipa_endpoint_tx_complete(endpoint, trans); 1367 else 1368 ipa_endpoint_rx_complete(endpoint, trans); 1369 } 1370 1371 void ipa_endpoint_trans_release(struct ipa_endpoint *endpoint, 1372 struct gsi_trans *trans) 1373 { 1374 if (endpoint->toward_ipa) { 1375 struct ipa *ipa = endpoint->ipa; 1376 1377 /* Nothing to do for command transactions */ 1378 if (endpoint != ipa->name_map[IPA_ENDPOINT_AP_COMMAND_TX]) { 1379 struct sk_buff *skb = trans->data; 1380 1381 if (skb) 1382 dev_kfree_skb_any(skb); 1383 } 1384 } else { 1385 struct page *page = trans->data; 1386 1387 if (page) { 1388 u32 buffer_size = endpoint->data->rx.buffer_size; 1389 1390 __free_pages(page, get_order(buffer_size)); 1391 } 1392 } 1393 } 1394 1395 void ipa_endpoint_default_route_set(struct ipa *ipa, u32 endpoint_id) 1396 { 1397 u32 val; 1398 1399 /* ROUTE_DIS is 0 */ 1400 val = u32_encode_bits(endpoint_id, ROUTE_DEF_PIPE_FMASK); 1401 val |= ROUTE_DEF_HDR_TABLE_FMASK; 1402 val |= u32_encode_bits(0, ROUTE_DEF_HDR_OFST_FMASK); 1403 val |= u32_encode_bits(endpoint_id, ROUTE_FRAG_DEF_PIPE_FMASK); 1404 val |= ROUTE_DEF_RETAIN_HDR_FMASK; 1405 1406 iowrite32(val, ipa->reg_virt + IPA_REG_ROUTE_OFFSET); 1407 } 1408 1409 void ipa_endpoint_default_route_clear(struct ipa *ipa) 1410 { 1411 ipa_endpoint_default_route_set(ipa, 0); 1412 } 1413 1414 /** 1415 * ipa_endpoint_reset_rx_aggr() - Reset RX endpoint with aggregation active 1416 * @endpoint: Endpoint to be reset 1417 * 1418 * If aggregation is active on an RX endpoint when a reset is performed 1419 * on its underlying GSI channel, a special sequence of actions must be 1420 * taken to ensure the IPA pipeline is properly cleared. 1421 * 1422 * Return: 0 if successful, or a negative error code 1423 */ 1424 static int ipa_endpoint_reset_rx_aggr(struct ipa_endpoint *endpoint) 1425 { 1426 struct device *dev = &endpoint->ipa->pdev->dev; 1427 struct ipa *ipa = endpoint->ipa; 1428 struct gsi *gsi = &ipa->gsi; 1429 bool suspended = false; 1430 dma_addr_t addr; 1431 u32 retries; 1432 u32 len = 1; 1433 void *virt; 1434 int ret; 1435 1436 virt = kzalloc(len, GFP_KERNEL); 1437 if (!virt) 1438 return -ENOMEM; 1439 1440 addr = dma_map_single(dev, virt, len, DMA_FROM_DEVICE); 1441 if (dma_mapping_error(dev, addr)) { 1442 ret = -ENOMEM; 1443 goto out_kfree; 1444 } 1445 1446 /* Force close aggregation before issuing the reset */ 1447 ipa_endpoint_force_close(endpoint); 1448 1449 /* Reset and reconfigure the channel with the doorbell engine 1450 * disabled. Then poll until we know aggregation is no longer 1451 * active. We'll re-enable the doorbell (if appropriate) when 1452 * we reset again below. 1453 */ 1454 gsi_channel_reset(gsi, endpoint->channel_id, false); 1455 1456 /* Make sure the channel isn't suspended */ 1457 suspended = ipa_endpoint_program_suspend(endpoint, false); 1458 1459 /* Start channel and do a 1 byte read */ 1460 ret = gsi_channel_start(gsi, endpoint->channel_id); 1461 if (ret) 1462 goto out_suspend_again; 1463 1464 ret = gsi_trans_read_byte(gsi, endpoint->channel_id, addr); 1465 if (ret) 1466 goto err_endpoint_stop; 1467 1468 /* Wait for aggregation to be closed on the channel */ 1469 retries = IPA_ENDPOINT_RESET_AGGR_RETRY_MAX; 1470 do { 1471 if (!ipa_endpoint_aggr_active(endpoint)) 1472 break; 1473 usleep_range(USEC_PER_MSEC, 2 * USEC_PER_MSEC); 1474 } while (retries--); 1475 1476 /* Check one last time */ 1477 if (ipa_endpoint_aggr_active(endpoint)) 1478 dev_err(dev, "endpoint %u still active during reset\n", 1479 endpoint->endpoint_id); 1480 1481 gsi_trans_read_byte_done(gsi, endpoint->channel_id); 1482 1483 ret = gsi_channel_stop(gsi, endpoint->channel_id); 1484 if (ret) 1485 goto out_suspend_again; 1486 1487 /* Finally, reset and reconfigure the channel again (re-enabling 1488 * the doorbell engine if appropriate). Sleep for 1 millisecond to 1489 * complete the channel reset sequence. Finish by suspending the 1490 * channel again (if necessary). 1491 */ 1492 gsi_channel_reset(gsi, endpoint->channel_id, true); 1493 1494 usleep_range(USEC_PER_MSEC, 2 * USEC_PER_MSEC); 1495 1496 goto out_suspend_again; 1497 1498 err_endpoint_stop: 1499 (void)gsi_channel_stop(gsi, endpoint->channel_id); 1500 out_suspend_again: 1501 if (suspended) 1502 (void)ipa_endpoint_program_suspend(endpoint, true); 1503 dma_unmap_single(dev, addr, len, DMA_FROM_DEVICE); 1504 out_kfree: 1505 kfree(virt); 1506 1507 return ret; 1508 } 1509 1510 static void ipa_endpoint_reset(struct ipa_endpoint *endpoint) 1511 { 1512 u32 channel_id = endpoint->channel_id; 1513 struct ipa *ipa = endpoint->ipa; 1514 bool special; 1515 int ret = 0; 1516 1517 /* On IPA v3.5.1, if an RX endpoint is reset while aggregation 1518 * is active, we need to handle things specially to recover. 1519 * All other cases just need to reset the underlying GSI channel. 1520 */ 1521 special = ipa->version < IPA_VERSION_4_0 && !endpoint->toward_ipa && 1522 endpoint->data->aggregation; 1523 if (special && ipa_endpoint_aggr_active(endpoint)) 1524 ret = ipa_endpoint_reset_rx_aggr(endpoint); 1525 else 1526 gsi_channel_reset(&ipa->gsi, channel_id, true); 1527 1528 if (ret) 1529 dev_err(&ipa->pdev->dev, 1530 "error %d resetting channel %u for endpoint %u\n", 1531 ret, endpoint->channel_id, endpoint->endpoint_id); 1532 } 1533 1534 static void ipa_endpoint_program(struct ipa_endpoint *endpoint) 1535 { 1536 if (endpoint->toward_ipa) { 1537 /* Newer versions of IPA use GSI channel flow control 1538 * instead of endpoint DELAY mode to prevent sending data. 1539 * Flow control is disabled for newly-allocated channels, 1540 * and we can assume flow control is not (ever) enabled 1541 * for AP TX channels. 1542 */ 1543 if (endpoint->ipa->version < IPA_VERSION_4_2) 1544 ipa_endpoint_program_delay(endpoint, false); 1545 } else { 1546 /* Ensure suspend mode is off on all AP RX endpoints */ 1547 (void)ipa_endpoint_program_suspend(endpoint, false); 1548 } 1549 ipa_endpoint_init_cfg(endpoint); 1550 ipa_endpoint_init_nat(endpoint); 1551 ipa_endpoint_init_hdr(endpoint); 1552 ipa_endpoint_init_hdr_ext(endpoint); 1553 ipa_endpoint_init_hdr_metadata_mask(endpoint); 1554 ipa_endpoint_init_mode(endpoint); 1555 ipa_endpoint_init_aggr(endpoint); 1556 if (!endpoint->toward_ipa) 1557 ipa_endpoint_init_hol_block_disable(endpoint); 1558 ipa_endpoint_init_deaggr(endpoint); 1559 ipa_endpoint_init_rsrc_grp(endpoint); 1560 ipa_endpoint_init_seq(endpoint); 1561 ipa_endpoint_status(endpoint); 1562 } 1563 1564 int ipa_endpoint_enable_one(struct ipa_endpoint *endpoint) 1565 { 1566 struct ipa *ipa = endpoint->ipa; 1567 struct gsi *gsi = &ipa->gsi; 1568 int ret; 1569 1570 ret = gsi_channel_start(gsi, endpoint->channel_id); 1571 if (ret) { 1572 dev_err(&ipa->pdev->dev, 1573 "error %d starting %cX channel %u for endpoint %u\n", 1574 ret, endpoint->toward_ipa ? 'T' : 'R', 1575 endpoint->channel_id, endpoint->endpoint_id); 1576 return ret; 1577 } 1578 1579 if (!endpoint->toward_ipa) { 1580 ipa_interrupt_suspend_enable(ipa->interrupt, 1581 endpoint->endpoint_id); 1582 ipa_endpoint_replenish_enable(endpoint); 1583 } 1584 1585 ipa->enabled |= BIT(endpoint->endpoint_id); 1586 1587 return 0; 1588 } 1589 1590 void ipa_endpoint_disable_one(struct ipa_endpoint *endpoint) 1591 { 1592 u32 mask = BIT(endpoint->endpoint_id); 1593 struct ipa *ipa = endpoint->ipa; 1594 struct gsi *gsi = &ipa->gsi; 1595 int ret; 1596 1597 if (!(ipa->enabled & mask)) 1598 return; 1599 1600 ipa->enabled ^= mask; 1601 1602 if (!endpoint->toward_ipa) { 1603 ipa_endpoint_replenish_disable(endpoint); 1604 ipa_interrupt_suspend_disable(ipa->interrupt, 1605 endpoint->endpoint_id); 1606 } 1607 1608 /* Note that if stop fails, the channel's state is not well-defined */ 1609 ret = gsi_channel_stop(gsi, endpoint->channel_id); 1610 if (ret) 1611 dev_err(&ipa->pdev->dev, 1612 "error %d attempting to stop endpoint %u\n", ret, 1613 endpoint->endpoint_id); 1614 } 1615 1616 void ipa_endpoint_suspend_one(struct ipa_endpoint *endpoint) 1617 { 1618 struct device *dev = &endpoint->ipa->pdev->dev; 1619 struct gsi *gsi = &endpoint->ipa->gsi; 1620 int ret; 1621 1622 if (!(endpoint->ipa->enabled & BIT(endpoint->endpoint_id))) 1623 return; 1624 1625 if (!endpoint->toward_ipa) { 1626 ipa_endpoint_replenish_disable(endpoint); 1627 (void)ipa_endpoint_program_suspend(endpoint, true); 1628 } 1629 1630 ret = gsi_channel_suspend(gsi, endpoint->channel_id); 1631 if (ret) 1632 dev_err(dev, "error %d suspending channel %u\n", ret, 1633 endpoint->channel_id); 1634 } 1635 1636 void ipa_endpoint_resume_one(struct ipa_endpoint *endpoint) 1637 { 1638 struct device *dev = &endpoint->ipa->pdev->dev; 1639 struct gsi *gsi = &endpoint->ipa->gsi; 1640 int ret; 1641 1642 if (!(endpoint->ipa->enabled & BIT(endpoint->endpoint_id))) 1643 return; 1644 1645 if (!endpoint->toward_ipa) 1646 (void)ipa_endpoint_program_suspend(endpoint, false); 1647 1648 ret = gsi_channel_resume(gsi, endpoint->channel_id); 1649 if (ret) 1650 dev_err(dev, "error %d resuming channel %u\n", ret, 1651 endpoint->channel_id); 1652 else if (!endpoint->toward_ipa) 1653 ipa_endpoint_replenish_enable(endpoint); 1654 } 1655 1656 void ipa_endpoint_suspend(struct ipa *ipa) 1657 { 1658 if (!ipa->setup_complete) 1659 return; 1660 1661 if (ipa->modem_netdev) 1662 ipa_modem_suspend(ipa->modem_netdev); 1663 1664 ipa_endpoint_suspend_one(ipa->name_map[IPA_ENDPOINT_AP_LAN_RX]); 1665 ipa_endpoint_suspend_one(ipa->name_map[IPA_ENDPOINT_AP_COMMAND_TX]); 1666 } 1667 1668 void ipa_endpoint_resume(struct ipa *ipa) 1669 { 1670 if (!ipa->setup_complete) 1671 return; 1672 1673 ipa_endpoint_resume_one(ipa->name_map[IPA_ENDPOINT_AP_COMMAND_TX]); 1674 ipa_endpoint_resume_one(ipa->name_map[IPA_ENDPOINT_AP_LAN_RX]); 1675 1676 if (ipa->modem_netdev) 1677 ipa_modem_resume(ipa->modem_netdev); 1678 } 1679 1680 static void ipa_endpoint_setup_one(struct ipa_endpoint *endpoint) 1681 { 1682 struct gsi *gsi = &endpoint->ipa->gsi; 1683 u32 channel_id = endpoint->channel_id; 1684 1685 /* Only AP endpoints get set up */ 1686 if (endpoint->ee_id != GSI_EE_AP) 1687 return; 1688 1689 endpoint->trans_tre_max = gsi_channel_trans_tre_max(gsi, channel_id); 1690 if (!endpoint->toward_ipa) { 1691 /* RX transactions require a single TRE, so the maximum 1692 * backlog is the same as the maximum outstanding TREs. 1693 */ 1694 clear_bit(IPA_REPLENISH_ENABLED, endpoint->replenish_flags); 1695 clear_bit(IPA_REPLENISH_ACTIVE, endpoint->replenish_flags); 1696 INIT_DELAYED_WORK(&endpoint->replenish_work, 1697 ipa_endpoint_replenish_work); 1698 } 1699 1700 ipa_endpoint_program(endpoint); 1701 1702 endpoint->ipa->set_up |= BIT(endpoint->endpoint_id); 1703 } 1704 1705 static void ipa_endpoint_teardown_one(struct ipa_endpoint *endpoint) 1706 { 1707 endpoint->ipa->set_up &= ~BIT(endpoint->endpoint_id); 1708 1709 if (!endpoint->toward_ipa) 1710 cancel_delayed_work_sync(&endpoint->replenish_work); 1711 1712 ipa_endpoint_reset(endpoint); 1713 } 1714 1715 void ipa_endpoint_setup(struct ipa *ipa) 1716 { 1717 u32 initialized = ipa->initialized; 1718 1719 ipa->set_up = 0; 1720 while (initialized) { 1721 u32 endpoint_id = __ffs(initialized); 1722 1723 initialized ^= BIT(endpoint_id); 1724 1725 ipa_endpoint_setup_one(&ipa->endpoint[endpoint_id]); 1726 } 1727 } 1728 1729 void ipa_endpoint_teardown(struct ipa *ipa) 1730 { 1731 u32 set_up = ipa->set_up; 1732 1733 while (set_up) { 1734 u32 endpoint_id = __fls(set_up); 1735 1736 set_up ^= BIT(endpoint_id); 1737 1738 ipa_endpoint_teardown_one(&ipa->endpoint[endpoint_id]); 1739 } 1740 ipa->set_up = 0; 1741 } 1742 1743 int ipa_endpoint_config(struct ipa *ipa) 1744 { 1745 struct device *dev = &ipa->pdev->dev; 1746 u32 initialized; 1747 u32 rx_base; 1748 u32 rx_mask; 1749 u32 tx_mask; 1750 int ret = 0; 1751 u32 max; 1752 u32 val; 1753 1754 /* Prior to IPAv3.5, the FLAVOR_0 register was not supported. 1755 * Furthermore, the endpoints were not grouped such that TX 1756 * endpoint numbers started with 0 and RX endpoints had numbers 1757 * higher than all TX endpoints, so we can't do the simple 1758 * direction check used for newer hardware below. 1759 * 1760 * For hardware that doesn't support the FLAVOR_0 register, 1761 * just set the available mask to support any endpoint, and 1762 * assume the configuration is valid. 1763 */ 1764 if (ipa->version < IPA_VERSION_3_5) { 1765 ipa->available = ~0; 1766 return 0; 1767 } 1768 1769 /* Find out about the endpoints supplied by the hardware, and ensure 1770 * the highest one doesn't exceed the number we support. 1771 */ 1772 val = ioread32(ipa->reg_virt + IPA_REG_FLAVOR_0_OFFSET); 1773 1774 /* Our RX is an IPA producer */ 1775 rx_base = u32_get_bits(val, IPA_PROD_LOWEST_FMASK); 1776 max = rx_base + u32_get_bits(val, IPA_MAX_PROD_PIPES_FMASK); 1777 if (max > IPA_ENDPOINT_MAX) { 1778 dev_err(dev, "too many endpoints (%u > %u)\n", 1779 max, IPA_ENDPOINT_MAX); 1780 return -EINVAL; 1781 } 1782 rx_mask = GENMASK(max - 1, rx_base); 1783 1784 /* Our TX is an IPA consumer */ 1785 max = u32_get_bits(val, IPA_MAX_CONS_PIPES_FMASK); 1786 tx_mask = GENMASK(max - 1, 0); 1787 1788 ipa->available = rx_mask | tx_mask; 1789 1790 /* Check for initialized endpoints not supported by the hardware */ 1791 if (ipa->initialized & ~ipa->available) { 1792 dev_err(dev, "unavailable endpoint id(s) 0x%08x\n", 1793 ipa->initialized & ~ipa->available); 1794 ret = -EINVAL; /* Report other errors too */ 1795 } 1796 1797 initialized = ipa->initialized; 1798 while (initialized) { 1799 u32 endpoint_id = __ffs(initialized); 1800 struct ipa_endpoint *endpoint; 1801 1802 initialized ^= BIT(endpoint_id); 1803 1804 /* Make sure it's pointing in the right direction */ 1805 endpoint = &ipa->endpoint[endpoint_id]; 1806 if ((endpoint_id < rx_base) != endpoint->toward_ipa) { 1807 dev_err(dev, "endpoint id %u wrong direction\n", 1808 endpoint_id); 1809 ret = -EINVAL; 1810 } 1811 } 1812 1813 return ret; 1814 } 1815 1816 void ipa_endpoint_deconfig(struct ipa *ipa) 1817 { 1818 ipa->available = 0; /* Nothing more to do */ 1819 } 1820 1821 static void ipa_endpoint_init_one(struct ipa *ipa, enum ipa_endpoint_name name, 1822 const struct ipa_gsi_endpoint_data *data) 1823 { 1824 struct ipa_endpoint *endpoint; 1825 1826 endpoint = &ipa->endpoint[data->endpoint_id]; 1827 1828 if (data->ee_id == GSI_EE_AP) 1829 ipa->channel_map[data->channel_id] = endpoint; 1830 ipa->name_map[name] = endpoint; 1831 1832 endpoint->ipa = ipa; 1833 endpoint->ee_id = data->ee_id; 1834 endpoint->channel_id = data->channel_id; 1835 endpoint->endpoint_id = data->endpoint_id; 1836 endpoint->toward_ipa = data->toward_ipa; 1837 endpoint->data = &data->endpoint.config; 1838 1839 ipa->initialized |= BIT(endpoint->endpoint_id); 1840 } 1841 1842 static void ipa_endpoint_exit_one(struct ipa_endpoint *endpoint) 1843 { 1844 endpoint->ipa->initialized &= ~BIT(endpoint->endpoint_id); 1845 1846 memset(endpoint, 0, sizeof(*endpoint)); 1847 } 1848 1849 void ipa_endpoint_exit(struct ipa *ipa) 1850 { 1851 u32 initialized = ipa->initialized; 1852 1853 while (initialized) { 1854 u32 endpoint_id = __fls(initialized); 1855 1856 initialized ^= BIT(endpoint_id); 1857 1858 ipa_endpoint_exit_one(&ipa->endpoint[endpoint_id]); 1859 } 1860 memset(ipa->name_map, 0, sizeof(ipa->name_map)); 1861 memset(ipa->channel_map, 0, sizeof(ipa->channel_map)); 1862 } 1863 1864 /* Returns a bitmask of endpoints that support filtering, or 0 on error */ 1865 u32 ipa_endpoint_init(struct ipa *ipa, u32 count, 1866 const struct ipa_gsi_endpoint_data *data) 1867 { 1868 enum ipa_endpoint_name name; 1869 u32 filter_map; 1870 1871 BUILD_BUG_ON(!IPA_REPLENISH_BATCH); 1872 1873 if (!ipa_endpoint_data_valid(ipa, count, data)) 1874 return 0; /* Error */ 1875 1876 ipa->initialized = 0; 1877 1878 filter_map = 0; 1879 for (name = 0; name < count; name++, data++) { 1880 if (ipa_gsi_endpoint_data_empty(data)) 1881 continue; /* Skip over empty slots */ 1882 1883 ipa_endpoint_init_one(ipa, name, data); 1884 1885 if (data->endpoint.filter_support) 1886 filter_map |= BIT(data->endpoint_id); 1887 } 1888 1889 if (!ipa_filter_map_valid(ipa, filter_map)) 1890 goto err_endpoint_exit; 1891 1892 return filter_map; /* Non-zero bitmask */ 1893 1894 err_endpoint_exit: 1895 ipa_endpoint_exit(ipa); 1896 1897 return 0; /* Error */ 1898 } 1899