Lines Matching +full:ipa +full:- +full:ap +full:- +full:to +full:- +full:modem
1 // SPDX-License-Identifier: GPL-2.0
3 /* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved.
4 * Copyright (C) 2019-2023 Linaro Ltd.
12 #include <linux/dma-direction.h>
16 #include "ipa.h"
27 #define IPA_REPLENISH_BATCH 16 /* Must be non-zero */
30 #define IPA_RX_BUFFER_OVERHEAD (PAGE_SIZE - SKB_MAX_ORDER(NET_SKB_PAD, 0))
32 /* Where to find the QMAP mux_id for a packet within modem-supplied metadata */
37 /** enum ipa_status_opcode - IPA status opcode field hardware values */
48 /** enum ipa_status_exception - IPA status exception field hardware values */
64 /** enum ipa_status_mask - IPA status mask field bitmask hardware values */
84 /* Special IPA filter/router rule field value indicating "rule miss" */
85 #define IPA_STATUS_RULE_MISS 0x3ff /* 10-bit filter/router rule fields */
87 /** The IPA status nat_type field uses enum ipa_nat_type hardware values */
89 /* enum ipa_status_field_id - IPA packet status structure field identifiers */
111 STATUS_TAG_LOW32, /* Low-order 32 bits of 48-bit tag */
112 STATUS_TAG_HIGH16, /* High-order 16 bits of 48-bit tag */
121 /* Size in bytes of an IPA packet status structure */
124 /* IPA status structure decoder; looks up field values for a structure */
125 static u32 ipa_status_extract(struct ipa *ipa, const void *data, in ipa_status_extract() argument
128 enum ipa_version version = ipa->version; in ipa_status_extract()
144 /* Status word 1, bits 21-23 are reserved (not IPA v5.0+) */ in ipa_status_extract()
145 /* Status word 1, bits 24-26 are reserved (IPA v5.0+) */ in ipa_status_extract()
150 /* Status word 1, bits 29-31 are reserved */ in ipa_status_extract()
163 /* ROUTER_TABLE is in word 3, bits 14-21 (IPA v5.0+) */ in ipa_status_extract()
205 /* Status word 7, bits 16-30 are reserved */ in ipa_status_extract()
206 /* Status word 7, bit 31 is reserved (not IPA v5.0+) */ in ipa_status_extract()
213 /* Compute the aggregation size value to use for a given buffer size */
221 * after that limit to receive a full MTU of data plus overhead. in ipa_aggr_size_kb()
224 rx_buffer_size -= IPA_MTU + IPA_RX_BUFFER_OVERHEAD; in ipa_aggr_size_kb()
231 static bool ipa_endpoint_data_valid_one(struct ipa *ipa, u32 count, in ipa_endpoint_data_valid_one() argument
236 struct device *dev = &ipa->pdev->dev; in ipa_endpoint_data_valid_one()
242 if (!data->toward_ipa) { in ipa_endpoint_data_valid_one()
249 if (data->endpoint.filter_support) { in ipa_endpoint_data_valid_one()
252 data->endpoint_id); in ipa_endpoint_data_valid_one()
256 /* Nothing more to check for non-AP RX */ in ipa_endpoint_data_valid_one()
257 if (data->ee_id != GSI_EE_AP) in ipa_endpoint_data_valid_one()
260 rx_config = &data->endpoint.config.rx; in ipa_endpoint_data_valid_one()
263 buffer_size = rx_config->buffer_size; in ipa_endpoint_data_valid_one()
267 data->endpoint_id, buffer_size, limit); in ipa_endpoint_data_valid_one()
271 if (!data->endpoint.config.aggregation) { in ipa_endpoint_data_valid_one()
275 if (rx_config->aggr_time_limit) { in ipa_endpoint_data_valid_one()
278 data->endpoint_id); in ipa_endpoint_data_valid_one()
282 if (rx_config->aggr_hard_limit) { in ipa_endpoint_data_valid_one()
284 data->endpoint_id); in ipa_endpoint_data_valid_one()
288 if (rx_config->aggr_close_eof) { in ipa_endpoint_data_valid_one()
290 data->endpoint_id); in ipa_endpoint_data_valid_one()
294 return result; /* Nothing more to check */ in ipa_endpoint_data_valid_one()
303 aggr_size = ipa_aggr_size_kb(buffer_size - NET_SKB_PAD, in ipa_endpoint_data_valid_one()
304 rx_config->aggr_hard_limit); in ipa_endpoint_data_valid_one()
305 reg = ipa_reg(ipa, ENDP_INIT_AGGR); in ipa_endpoint_data_valid_one()
310 data->endpoint_id, aggr_size, limit); in ipa_endpoint_data_valid_one()
315 return true; /* Nothing more to check for RX */ in ipa_endpoint_data_valid_one()
318 /* Starting with IPA v4.5 sequencer replication is obsolete */ in ipa_endpoint_data_valid_one()
319 if (ipa->version >= IPA_VERSION_4_5) { in ipa_endpoint_data_valid_one()
320 if (data->endpoint.config.tx.seq_rep_type) { in ipa_endpoint_data_valid_one()
321 dev_err(dev, "no-zero seq_rep_type TX endpoint %u\n", in ipa_endpoint_data_valid_one()
322 data->endpoint_id); in ipa_endpoint_data_valid_one()
327 if (data->endpoint.config.status_enable) { in ipa_endpoint_data_valid_one()
328 other_name = data->endpoint.config.tx.status_endpoint; in ipa_endpoint_data_valid_one()
332 other_name, data->endpoint_id); in ipa_endpoint_data_valid_one()
341 other_name, data->endpoint_id); in ipa_endpoint_data_valid_one()
345 /* ...and has to be an RX endpoint... */ in ipa_endpoint_data_valid_one()
346 if (other_data->toward_ipa) { in ipa_endpoint_data_valid_one()
349 data->endpoint_id); in ipa_endpoint_data_valid_one()
353 /* ...and if it's to be an AP endpoint... */ in ipa_endpoint_data_valid_one()
354 if (other_data->ee_id == GSI_EE_AP) { in ipa_endpoint_data_valid_one()
356 if (!other_data->endpoint.config.status_enable) { in ipa_endpoint_data_valid_one()
359 other_data->endpoint_id); in ipa_endpoint_data_valid_one()
365 if (data->endpoint.config.dma_mode) { in ipa_endpoint_data_valid_one()
366 other_name = data->endpoint.config.dma_endpoint; in ipa_endpoint_data_valid_one()
370 other_name, data->endpoint_id); in ipa_endpoint_data_valid_one()
378 other_name, data->endpoint_id); in ipa_endpoint_data_valid_one()
387 static u32 ipa_endpoint_max(struct ipa *ipa, u32 count, in ipa_endpoint_max() argument
391 struct device *dev = &ipa->pdev->dev; in ipa_endpoint_max()
411 dev_err(dev, "AP->modem TX endpoint not defined\n"); in ipa_endpoint_max()
415 dev_err(dev, "AP<-modem RX endpoint not defined\n"); in ipa_endpoint_max()
421 if (!ipa_endpoint_data_valid_one(ipa, count, data, dp)) in ipa_endpoint_max()
423 max = max_t(u32, max, dp->endpoint_id); in ipa_endpoint_max()
429 /* Allocate a transaction to use on a non-command endpoint */
433 struct gsi *gsi = &endpoint->ipa->gsi; in ipa_endpoint_trans_alloc()
434 u32 channel_id = endpoint->channel_id; in ipa_endpoint_trans_alloc()
437 direction = endpoint->toward_ipa ? DMA_TO_DEVICE : DMA_FROM_DEVICE; in ipa_endpoint_trans_alloc()
443 * Note that suspend is not supported starting with IPA v4.0, and
444 * delay mode should not be used starting with IPA v4.2.
449 struct ipa *ipa = endpoint->ipa; in ipa_endpoint_init_ctrl() local
457 if (endpoint->toward_ipa) in ipa_endpoint_init_ctrl()
458 WARN_ON(ipa->version >= IPA_VERSION_4_2); in ipa_endpoint_init_ctrl()
460 WARN_ON(ipa->version >= IPA_VERSION_4_0); in ipa_endpoint_init_ctrl()
462 reg = ipa_reg(ipa, ENDP_INIT_CTRL); in ipa_endpoint_init_ctrl()
463 offset = reg_n_offset(reg, endpoint->endpoint_id); in ipa_endpoint_init_ctrl()
464 val = ioread32(ipa->reg_virt + offset); in ipa_endpoint_init_ctrl()
466 field_id = endpoint->toward_ipa ? ENDP_DELAY : ENDP_SUSPEND; in ipa_endpoint_init_ctrl()
474 iowrite32(val, ipa->reg_virt + offset); in ipa_endpoint_init_ctrl()
484 /* Delay mode should not be used for IPA v4.2+ */ in ipa_endpoint_program_delay()
485 WARN_ON(endpoint->ipa->version >= IPA_VERSION_4_2); in ipa_endpoint_program_delay()
486 WARN_ON(!endpoint->toward_ipa); in ipa_endpoint_program_delay()
493 u32 endpoint_id = endpoint->endpoint_id; in ipa_endpoint_aggr_active()
494 struct ipa *ipa = endpoint->ipa; in ipa_endpoint_aggr_active() local
499 WARN_ON(!test_bit(endpoint_id, ipa->available)); in ipa_endpoint_aggr_active()
501 reg = ipa_reg(ipa, STATE_AGGR_ACTIVE); in ipa_endpoint_aggr_active()
502 val = ioread32(ipa->reg_virt + reg_n_offset(reg, unit)); in ipa_endpoint_aggr_active()
509 u32 endpoint_id = endpoint->endpoint_id; in ipa_endpoint_force_close()
511 struct ipa *ipa = endpoint->ipa; in ipa_endpoint_force_close() local
515 WARN_ON(!test_bit(endpoint_id, ipa->available)); in ipa_endpoint_force_close()
517 reg = ipa_reg(ipa, AGGR_FORCE_CLOSE); in ipa_endpoint_force_close()
518 iowrite32(mask, ipa->reg_virt + reg_n_offset(reg, unit)); in ipa_endpoint_force_close()
522 * ipa_endpoint_suspend_aggr() - Emulate suspend interrupt
523 * @endpoint: Endpoint on which to emulate a suspend
525 * Emulate suspend IPA interrupt to unsuspend an endpoint suspended
526 * with an open aggregation frame. This is to work around a hardware
527 * issue in IPA version 3.5.1 where the suspend interrupt will not be
532 struct ipa *ipa = endpoint->ipa; in ipa_endpoint_suspend_aggr() local
534 if (!endpoint->config.aggregation) in ipa_endpoint_suspend_aggr()
537 /* Nothing to do if the endpoint doesn't have aggregation open */ in ipa_endpoint_suspend_aggr()
544 ipa_interrupt_simulate_suspend(ipa->interrupt); in ipa_endpoint_suspend_aggr()
553 if (endpoint->ipa->version >= IPA_VERSION_4_0) in ipa_endpoint_program_suspend()
554 return enable; /* For IPA v4.0+, no change made */ in ipa_endpoint_program_suspend()
556 WARN_ON(endpoint->toward_ipa); in ipa_endpoint_program_suspend()
561 * generate a SUSPEND IPA interrupt. If enabling suspend, have in ipa_endpoint_program_suspend()
570 /* Put all modem RX endpoints into suspend mode, and stop transmission
571 * on all modem TX endpoints. Prior to IPA v4.2, endpoint DELAY mode is
572 * used for TX endpoints; starting with IPA v4.2 we use GSI channel flow
575 void ipa_endpoint_modem_pause_all(struct ipa *ipa, bool enable) in ipa_endpoint_modem_pause_all() argument
579 while (endpoint_id < ipa->endpoint_count) { in ipa_endpoint_modem_pause_all()
580 struct ipa_endpoint *endpoint = &ipa->endpoint[endpoint_id++]; in ipa_endpoint_modem_pause_all()
582 if (endpoint->ee_id != GSI_EE_MODEM) in ipa_endpoint_modem_pause_all()
585 if (!endpoint->toward_ipa) in ipa_endpoint_modem_pause_all()
587 else if (ipa->version < IPA_VERSION_4_2) in ipa_endpoint_modem_pause_all()
590 gsi_modem_channel_flow_control(&ipa->gsi, in ipa_endpoint_modem_pause_all()
591 endpoint->channel_id, in ipa_endpoint_modem_pause_all()
596 /* Reset all modem endpoints to use the default exception endpoint */
597 int ipa_endpoint_modem_exception_reset_all(struct ipa *ipa) in ipa_endpoint_modem_exception_reset_all() argument
603 /* We need one command per modem TX endpoint, plus the commands in ipa_endpoint_modem_exception_reset_all()
606 count = ipa->modem_tx_count + ipa_cmd_pipeline_clear_count(); in ipa_endpoint_modem_exception_reset_all()
607 trans = ipa_cmd_trans_alloc(ipa, count); in ipa_endpoint_modem_exception_reset_all()
609 dev_err(&ipa->pdev->dev, in ipa_endpoint_modem_exception_reset_all()
610 "no transaction to reset modem exception endpoints\n"); in ipa_endpoint_modem_exception_reset_all()
611 return -EBUSY; in ipa_endpoint_modem_exception_reset_all()
614 for_each_set_bit(endpoint_id, ipa->defined, ipa->endpoint_count) { in ipa_endpoint_modem_exception_reset_all()
619 /* We only reset modem TX endpoints */ in ipa_endpoint_modem_exception_reset_all()
620 endpoint = &ipa->endpoint[endpoint_id]; in ipa_endpoint_modem_exception_reset_all()
621 if (!(endpoint->ee_id == GSI_EE_MODEM && endpoint->toward_ipa)) in ipa_endpoint_modem_exception_reset_all()
624 reg = ipa_reg(ipa, ENDP_STATUS); in ipa_endpoint_modem_exception_reset_all()
638 ipa_cmd_pipeline_clear_wait(ipa); in ipa_endpoint_modem_exception_reset_all()
645 u32 endpoint_id = endpoint->endpoint_id; in ipa_endpoint_init_cfg()
646 struct ipa *ipa = endpoint->ipa; in ipa_endpoint_init_cfg() local
651 reg = ipa_reg(ipa, ENDP_INIT_CFG); in ipa_endpoint_init_cfg()
653 if (endpoint->config.checksum) { in ipa_endpoint_init_cfg()
654 enum ipa_version version = ipa->version; in ipa_endpoint_init_cfg()
656 if (endpoint->toward_ipa) { in ipa_endpoint_init_cfg()
659 /* Checksum header offset is in 4-byte units */ in ipa_endpoint_init_cfg()
677 iowrite32(val, ipa->reg_virt + reg_n_offset(reg, endpoint_id)); in ipa_endpoint_init_cfg()
682 u32 endpoint_id = endpoint->endpoint_id; in ipa_endpoint_init_nat()
683 struct ipa *ipa = endpoint->ipa; in ipa_endpoint_init_nat() local
687 if (!endpoint->toward_ipa) in ipa_endpoint_init_nat()
690 reg = ipa_reg(ipa, ENDP_INIT_NAT); in ipa_endpoint_init_nat()
693 iowrite32(val, ipa->reg_virt + reg_n_offset(reg, endpoint_id)); in ipa_endpoint_init_nat()
702 if (!endpoint->config.checksum) in ipa_qmap_header_size()
706 /* Checksum header inserted for AP TX endpoints only */ in ipa_qmap_header_size()
707 if (endpoint->toward_ipa) in ipa_qmap_header_size()
724 /* We know field_max can be used as a mask (2^n - 1) */ in ipa_header_size_encode()
731 /* IPA v4.5 adds a few more most-significant bits */ in ipa_header_size_encode()
746 /* We know field_max can be used as a mask (2^n - 1) */ in ipa_metadata_offset_encode()
753 /* IPA v4.5 adds a few more most-significant bits */ in ipa_metadata_offset_encode()
762 * ipa_endpoint_init_hdr() - Initialize HDR endpoint configuration register
766 * header structure. The QMAP header contains a 1-byte mux_id and 2-byte
767 * packet size field, and we have the IPA hardware populate both for each
769 * to use big endian format.
774 * The mux_id comes from a 4-byte metadata value supplied with each packet
775 * by the modem. It is *not* a QMAP header, but it does contain the mux_id
776 * value that we want, in its low-order byte. A bitmask defined in the
777 * endpoint's METADATA_MASK register defines which byte within the modem
784 u32 endpoint_id = endpoint->endpoint_id; in ipa_endpoint_init_hdr()
785 struct ipa *ipa = endpoint->ipa; in ipa_endpoint_init_hdr() local
789 reg = ipa_reg(ipa, ENDP_INIT_HDR); in ipa_endpoint_init_hdr()
790 if (endpoint->config.qmap) { in ipa_endpoint_init_hdr()
791 enum ipa_version version = ipa->version; in ipa_endpoint_init_hdr()
797 /* Define how to fill fields in a received QMAP header */ in ipa_endpoint_init_hdr()
798 if (!endpoint->toward_ipa) { in ipa_endpoint_init_hdr()
801 /* Where IPA will write the metadata value */ in ipa_endpoint_init_hdr()
805 /* Where IPA will write the length */ in ipa_endpoint_init_hdr()
807 /* Upper bits are stored in HDR_EXT with IPA v4.5 */ in ipa_endpoint_init_hdr()
814 /* For QMAP TX, metadata offset is 0 (modem assumes this) */ in ipa_endpoint_init_hdr()
823 iowrite32(val, ipa->reg_virt + reg_n_offset(reg, endpoint_id)); in ipa_endpoint_init_hdr()
828 u32 pad_align = endpoint->config.rx.pad_align; in ipa_endpoint_init_hdr_ext()
829 u32 endpoint_id = endpoint->endpoint_id; in ipa_endpoint_init_hdr_ext()
830 struct ipa *ipa = endpoint->ipa; in ipa_endpoint_init_hdr_ext() local
834 reg = ipa_reg(ipa, ENDP_INIT_HDR_EXT); in ipa_endpoint_init_hdr_ext()
835 if (endpoint->config.qmap) { in ipa_endpoint_init_hdr_ext()
846 if (!endpoint->toward_ipa) { in ipa_endpoint_init_hdr_ext()
855 if (!endpoint->toward_ipa) in ipa_endpoint_init_hdr_ext()
858 /* IPA v4.5 adds some most-significant bits to a few fields, in ipa_endpoint_init_hdr_ext()
861 if (ipa->version >= IPA_VERSION_4_5) { in ipa_endpoint_init_hdr_ext()
863 if (endpoint->config.qmap && !endpoint->toward_ipa) { in ipa_endpoint_init_hdr_ext()
875 iowrite32(val, ipa->reg_virt + reg_n_offset(reg, endpoint_id)); in ipa_endpoint_init_hdr_ext()
880 u32 endpoint_id = endpoint->endpoint_id; in ipa_endpoint_init_hdr_metadata_mask()
881 struct ipa *ipa = endpoint->ipa; in ipa_endpoint_init_hdr_metadata_mask() local
886 if (endpoint->toward_ipa) in ipa_endpoint_init_hdr_metadata_mask()
889 reg = ipa_reg(ipa, ENDP_INIT_HDR_METADATA_MASK); in ipa_endpoint_init_hdr_metadata_mask()
893 if (endpoint->config.qmap) in ipa_endpoint_init_hdr_metadata_mask()
896 iowrite32(val, ipa->reg_virt + offset); in ipa_endpoint_init_hdr_metadata_mask()
901 struct ipa *ipa = endpoint->ipa; in ipa_endpoint_init_mode() local
906 if (!endpoint->toward_ipa) in ipa_endpoint_init_mode()
909 reg = ipa_reg(ipa, ENDP_INIT_MODE); in ipa_endpoint_init_mode()
910 if (endpoint->config.dma_mode) { in ipa_endpoint_init_mode()
911 enum ipa_endpoint_name name = endpoint->config.dma_endpoint; in ipa_endpoint_init_mode()
912 u32 dma_endpoint_id = ipa->name_map[name]->endpoint_id; in ipa_endpoint_init_mode()
921 offset = reg_n_offset(reg, endpoint->endpoint_id); in ipa_endpoint_init_mode()
922 iowrite32(val, ipa->reg_virt + offset); in ipa_endpoint_init_mode()
925 /* For IPA v4.5+, times are expressed using Qtime. A time is represented
927 * ipa_qtime_config(). Three (or, starting with IPA v5.0, four) pulse
931 * available to the AP; a third is available starting with IPA v5.0.
933 * represents the time period provided, and returns the tick count to
934 * use to represent that time.
937 ipa_qtime_val(struct ipa *ipa, u32 microseconds, u32 max, u32 *select) in ipa_qtime_val() argument
953 if (ipa->version >= IPA_VERSION_5_0) { in ipa_qtime_val()
965 /* Encode the aggregation timer limit (microseconds) based on IPA version */
966 static u32 aggr_time_limit_encode(struct ipa *ipa, const struct reg *reg, in aggr_time_limit_encode() argument
973 return 0; /* Nothing to compute if time limit is 0 */ in aggr_time_limit_encode()
976 if (ipa->version >= IPA_VERSION_4_5) { in aggr_time_limit_encode()
979 ticks = ipa_qtime_val(ipa, microseconds, max, &select); in aggr_time_limit_encode()
995 u32 endpoint_id = endpoint->endpoint_id; in ipa_endpoint_init_aggr()
996 struct ipa *ipa = endpoint->ipa; in ipa_endpoint_init_aggr() local
1000 reg = ipa_reg(ipa, ENDP_INIT_AGGR); in ipa_endpoint_init_aggr()
1001 if (endpoint->config.aggregation) { in ipa_endpoint_init_aggr()
1002 if (!endpoint->toward_ipa) { in ipa_endpoint_init_aggr()
1007 rx_config = &endpoint->config.rx; in ipa_endpoint_init_aggr()
1011 buffer_size = rx_config->buffer_size; in ipa_endpoint_init_aggr()
1012 limit = ipa_aggr_size_kb(buffer_size - NET_SKB_PAD, in ipa_endpoint_init_aggr()
1013 rx_config->aggr_hard_limit); in ipa_endpoint_init_aggr()
1016 limit = rx_config->aggr_time_limit; in ipa_endpoint_init_aggr()
1017 val |= aggr_time_limit_encode(ipa, reg, limit); in ipa_endpoint_init_aggr()
1021 if (rx_config->aggr_close_eof) in ipa_endpoint_init_aggr()
1029 /* AGGR_GRAN_SEL is 0 for IPA v4.5 */ in ipa_endpoint_init_aggr()
1035 iowrite32(val, ipa->reg_virt + reg_n_offset(reg, endpoint_id)); in ipa_endpoint_init_aggr()
1038 /* The head-of-line blocking timer is defined as a tick count. For
1039 * IPA version 4.5 the tick count is based on the Qtimer, which is
1040 * derived from the 19.2 MHz SoC XO clock. For older IPA versions
1041 * each tick represents 128 cycles of the IPA core clock.
1044 * that should be written to the ENDP_INIT_HOL_BLOCK_TIMER register.
1046 static u32 hol_block_timer_encode(struct ipa *ipa, const struct reg *reg, in hol_block_timer_encode() argument
1057 return 0; /* Nothing to compute if timer period is 0 */ in hol_block_timer_encode()
1059 if (ipa->version >= IPA_VERSION_4_5) { in hol_block_timer_encode()
1064 ticks = ipa_qtime_val(ipa, microseconds, max, &select); in hol_block_timer_encode()
1070 /* Use 64 bit arithmetic to avoid overflow */ in hol_block_timer_encode()
1071 rate = ipa_core_clock_rate(ipa); in hol_block_timer_encode()
1074 /* We still need the result to fit into the field */ in hol_block_timer_encode()
1077 /* IPA v3.5.1 through v4.1 just record the tick count */ in hol_block_timer_encode()
1078 if (ipa->version < IPA_VERSION_4_2) in hol_block_timer_encode()
1081 /* For IPA v4.2, the tick count is represented by base and in hol_block_timer_encode()
1082 * scale fields within the 32-bit timer register, where: in hol_block_timer_encode()
1091 scale = high > width ? high - width : 0; in hol_block_timer_encode()
1093 /* If we're scaling, round up to get a closer result */ in hol_block_timer_encode()
1094 ticks += 1 << (scale - 1); in hol_block_timer_encode()
1110 u32 endpoint_id = endpoint->endpoint_id; in ipa_endpoint_init_hol_block_timer()
1111 struct ipa *ipa = endpoint->ipa; in ipa_endpoint_init_hol_block_timer() local
1116 reg = ipa_reg(ipa, ENDP_INIT_HOL_BLOCK_TIMER); in ipa_endpoint_init_hol_block_timer()
1117 val = hol_block_timer_encode(ipa, reg, microseconds); in ipa_endpoint_init_hol_block_timer()
1119 iowrite32(val, ipa->reg_virt + reg_n_offset(reg, endpoint_id)); in ipa_endpoint_init_hol_block_timer()
1125 u32 endpoint_id = endpoint->endpoint_id; in ipa_endpoint_init_hol_block_en()
1126 struct ipa *ipa = endpoint->ipa; in ipa_endpoint_init_hol_block_en() local
1131 reg = ipa_reg(ipa, ENDP_INIT_HOL_BLOCK_EN); in ipa_endpoint_init_hol_block_en()
1135 iowrite32(val, ipa->reg_virt + offset); in ipa_endpoint_init_hol_block_en()
1137 /* When enabling, the register must be written twice for IPA v4.5+ */ in ipa_endpoint_init_hol_block_en()
1138 if (enable && ipa->version >= IPA_VERSION_4_5) in ipa_endpoint_init_hol_block_en()
1139 iowrite32(val, ipa->reg_virt + offset); in ipa_endpoint_init_hol_block_en()
1155 void ipa_endpoint_modem_hol_block_clear_all(struct ipa *ipa) in ipa_endpoint_modem_hol_block_clear_all() argument
1159 while (endpoint_id < ipa->endpoint_count) { in ipa_endpoint_modem_hol_block_clear_all()
1160 struct ipa_endpoint *endpoint = &ipa->endpoint[endpoint_id++]; in ipa_endpoint_modem_hol_block_clear_all()
1162 if (endpoint->toward_ipa || endpoint->ee_id != GSI_EE_MODEM) in ipa_endpoint_modem_hol_block_clear_all()
1172 u32 endpoint_id = endpoint->endpoint_id; in ipa_endpoint_init_deaggr()
1173 struct ipa *ipa = endpoint->ipa; in ipa_endpoint_init_deaggr() local
1177 if (!endpoint->toward_ipa) in ipa_endpoint_init_deaggr()
1180 reg = ipa_reg(ipa, ENDP_INIT_DEAGGR); in ipa_endpoint_init_deaggr()
1186 iowrite32(val, ipa->reg_virt + reg_n_offset(reg, endpoint_id)); in ipa_endpoint_init_deaggr()
1191 u32 resource_group = endpoint->config.resource_group; in ipa_endpoint_init_rsrc_grp()
1192 u32 endpoint_id = endpoint->endpoint_id; in ipa_endpoint_init_rsrc_grp()
1193 struct ipa *ipa = endpoint->ipa; in ipa_endpoint_init_rsrc_grp() local
1197 reg = ipa_reg(ipa, ENDP_INIT_RSRC_GRP); in ipa_endpoint_init_rsrc_grp()
1200 iowrite32(val, ipa->reg_virt + reg_n_offset(reg, endpoint_id)); in ipa_endpoint_init_rsrc_grp()
1205 u32 endpoint_id = endpoint->endpoint_id; in ipa_endpoint_init_seq()
1206 struct ipa *ipa = endpoint->ipa; in ipa_endpoint_init_seq() local
1210 if (!endpoint->toward_ipa) in ipa_endpoint_init_seq()
1213 reg = ipa_reg(ipa, ENDP_INIT_SEQ); in ipa_endpoint_init_seq()
1215 /* Low-order byte configures primary packet processing */ in ipa_endpoint_init_seq()
1216 val = reg_encode(reg, SEQ_TYPE, endpoint->config.tx.seq_type); in ipa_endpoint_init_seq()
1219 if (ipa->version < IPA_VERSION_4_5) in ipa_endpoint_init_seq()
1221 endpoint->config.tx.seq_rep_type); in ipa_endpoint_init_seq()
1223 iowrite32(val, ipa->reg_virt + reg_n_offset(reg, endpoint_id)); in ipa_endpoint_init_seq()
1227 * ipa_endpoint_skb_tx() - Transmit a socket buffer
1229 * @skb: Socket buffer to send
1239 /* Make sure source endpoint's TLV FIFO has enough entries to in ipa_endpoint_skb_tx()
1243 nr_frags = skb_shinfo(skb)->nr_frags; in ipa_endpoint_skb_tx()
1244 if (nr_frags > endpoint->skb_frag_max) { in ipa_endpoint_skb_tx()
1246 return -E2BIG; in ipa_endpoint_skb_tx()
1252 return -EBUSY; in ipa_endpoint_skb_tx()
1257 trans->data = skb; /* transaction owns skb now */ in ipa_endpoint_skb_tx()
1266 return -ENOMEM; in ipa_endpoint_skb_tx()
1271 u32 endpoint_id = endpoint->endpoint_id; in ipa_endpoint_status()
1272 struct ipa *ipa = endpoint->ipa; in ipa_endpoint_status() local
1276 reg = ipa_reg(ipa, ENDP_STATUS); in ipa_endpoint_status()
1277 if (endpoint->config.status_enable) { in ipa_endpoint_status()
1279 if (endpoint->toward_ipa) { in ipa_endpoint_status()
1283 name = endpoint->config.tx.status_endpoint; in ipa_endpoint_status()
1284 status_endpoint_id = ipa->name_map[name]->endpoint_id; in ipa_endpoint_status()
1288 /* STATUS_LOCATION is 0, meaning IPA packet status in ipa_endpoint_status()
1289 * precedes the packet (not present for IPA v4.5+) in ipa_endpoint_status()
1294 iowrite32(val, ipa->reg_virt + reg_n_offset(reg, endpoint_id)); in ipa_endpoint_status()
1306 buffer_size = endpoint->config.rx.buffer_size; in ipa_endpoint_replenish_one()
1309 return -ENOMEM; in ipa_endpoint_replenish_one()
1311 /* Offset the buffer to make space for skb headroom */ in ipa_endpoint_replenish_one()
1313 len = buffer_size - offset; in ipa_endpoint_replenish_one()
1319 trans->data = page; /* transaction owns page now */ in ipa_endpoint_replenish_one()
1325 * ipa_endpoint_replenish() - Replenish endpoint receive buffers
1326 * @endpoint: Endpoint to be replenished
1328 * The IPA hardware can hold a fixed number of receive buffers for an RX
1330 * buffer. If an endpoint's "backlog" is non-zero, it indicates how many
1331 * more receive buffers can be supplied to the hardware. Replenishing for
1332 * an endpoint can be disabled, in which case buffers are not queued to
1339 if (!test_bit(IPA_REPLENISH_ENABLED, endpoint->replenish_flags)) in ipa_endpoint_replenish()
1343 if (test_and_set_bit(IPA_REPLENISH_ACTIVE, endpoint->replenish_flags)) in ipa_endpoint_replenish()
1354 doorbell = !(++endpoint->replenish_count % IPA_REPLENISH_BATCH); in ipa_endpoint_replenish()
1358 clear_bit(IPA_REPLENISH_ACTIVE, endpoint->replenish_flags); in ipa_endpoint_replenish()
1364 clear_bit(IPA_REPLENISH_ACTIVE, endpoint->replenish_flags); in ipa_endpoint_replenish()
1366 /* Whenever a receive buffer transaction completes we'll try to in ipa_endpoint_replenish()
1367 * replenish again. It's unlikely, but if we fail to supply even in ipa_endpoint_replenish()
1369 * If the hardware has no receive buffers queued, schedule work to in ipa_endpoint_replenish()
1372 if (gsi_channel_trans_idle(&endpoint->ipa->gsi, endpoint->channel_id)) in ipa_endpoint_replenish()
1373 schedule_delayed_work(&endpoint->replenish_work, in ipa_endpoint_replenish()
1379 set_bit(IPA_REPLENISH_ENABLED, endpoint->replenish_flags); in ipa_endpoint_replenish_enable()
1382 if (gsi_channel_trans_idle(&endpoint->ipa->gsi, endpoint->channel_id)) in ipa_endpoint_replenish_enable()
1388 clear_bit(IPA_REPLENISH_ENABLED, endpoint->replenish_flags); in ipa_endpoint_replenish_disable()
1406 if (!endpoint->netdev) in ipa_endpoint_skb_copy()
1413 memcpy(skb->data, data, len); in ipa_endpoint_skb_copy()
1414 skb->truesize += extra; in ipa_endpoint_skb_copy()
1417 ipa_modem_skb_rx(endpoint->netdev, skb); in ipa_endpoint_skb_copy()
1423 u32 buffer_size = endpoint->config.rx.buffer_size; in ipa_endpoint_skb_build()
1426 /* Nothing to do if there's no netdev */ in ipa_endpoint_skb_build()
1427 if (!endpoint->netdev) in ipa_endpoint_skb_build()
1430 WARN_ON(len > SKB_WITH_OVERHEAD(buffer_size - NET_SKB_PAD)); in ipa_endpoint_skb_build()
1439 /* Receive the buffer (or record drop if unable to build it) */ in ipa_endpoint_skb_build()
1440 ipa_modem_skb_rx(endpoint->netdev, skb); in ipa_endpoint_skb_build()
1445 /* The format of an IPA packet status structure is the same for several
1464 struct ipa *ipa = endpoint->ipa; in ipa_endpoint_status_skip() local
1468 opcode = ipa_status_extract(ipa, data, STATUS_OPCODE); in ipa_endpoint_status_skip()
1472 endpoint_id = ipa_status_extract(ipa, data, STATUS_DST_ENDPOINT); in ipa_endpoint_status_skip()
1473 if (endpoint_id != endpoint->endpoint_id) in ipa_endpoint_status_skip()
1484 struct ipa *ipa = endpoint->ipa; in ipa_endpoint_status_tag_valid() local
1487 status_mask = ipa_status_extract(ipa, data, STATUS_MASK); in ipa_endpoint_status_tag_valid()
1491 /* The status contains a valid tag. We know the packet was sent to in ipa_endpoint_status_tag_valid()
1493 * If the packet came from the AP->command TX endpoint we know in ipa_endpoint_status_tag_valid()
1496 endpoint_id = ipa_status_extract(ipa, data, STATUS_SRC_ENDPOINT); in ipa_endpoint_status_tag_valid()
1497 command_endpoint = ipa->name_map[IPA_ENDPOINT_AP_COMMAND_TX]; in ipa_endpoint_status_tag_valid()
1498 if (endpoint_id == command_endpoint->endpoint_id) { in ipa_endpoint_status_tag_valid()
1499 complete(&ipa->completion); in ipa_endpoint_status_tag_valid()
1501 dev_err(&ipa->pdev->dev, in ipa_endpoint_status_tag_valid()
1514 struct ipa *ipa = endpoint->ipa; in ipa_endpoint_status_drop() local
1522 exception = ipa_status_extract(ipa, data, STATUS_EXCEPTION); in ipa_endpoint_status_drop()
1526 /* Drop the packet if it fails to match a routing rule; otherwise no */ in ipa_endpoint_status_drop()
1527 rule = ipa_status_extract(ipa, data, STATUS_ROUTER_RULE_INDEX); in ipa_endpoint_status_drop()
1535 u32 buffer_size = endpoint->config.rx.buffer_size; in ipa_endpoint_status_parse()
1537 u32 unused = buffer_size - total_len; in ipa_endpoint_status_parse()
1538 struct ipa *ipa = endpoint->ipa; in ipa_endpoint_status_parse() local
1547 dev_err(&endpoint->ipa->pdev->dev, in ipa_endpoint_status_parse()
1554 length = ipa_status_extract(ipa, data, STATUS_LENGTH); in ipa_endpoint_status_parse()
1557 resid -= IPA_STATUS_SIZE; in ipa_endpoint_status_parse()
1562 * including the status. If the hardware is configured to in ipa_endpoint_status_parse()
1563 * pad packet data to an aligned boundary, account for that. in ipa_endpoint_status_parse()
1567 align = endpoint->config.rx.pad_align ? : 1; in ipa_endpoint_status_parse()
1569 if (endpoint->config.checksum) in ipa_endpoint_status_parse()
1590 resid -= len; in ipa_endpoint_status_parse()
1599 if (endpoint->toward_ipa) in ipa_endpoint_trans_complete()
1602 if (trans->cancelled) in ipa_endpoint_trans_complete()
1606 page = trans->data; in ipa_endpoint_trans_complete()
1607 if (endpoint->config.status_enable) in ipa_endpoint_trans_complete()
1608 ipa_endpoint_status_parse(endpoint, page, trans->len); in ipa_endpoint_trans_complete()
1609 else if (ipa_endpoint_skb_build(endpoint, page, trans->len)) in ipa_endpoint_trans_complete()
1610 trans->data = NULL; /* Pages have been consumed */ in ipa_endpoint_trans_complete()
1618 if (endpoint->toward_ipa) { in ipa_endpoint_trans_release()
1619 struct ipa *ipa = endpoint->ipa; in ipa_endpoint_trans_release() local
1621 /* Nothing to do for command transactions */ in ipa_endpoint_trans_release()
1622 if (endpoint != ipa->name_map[IPA_ENDPOINT_AP_COMMAND_TX]) { in ipa_endpoint_trans_release()
1623 struct sk_buff *skb = trans->data; in ipa_endpoint_trans_release()
1629 struct page *page = trans->data; in ipa_endpoint_trans_release()
1636 void ipa_endpoint_default_route_set(struct ipa *ipa, u32 endpoint_id) in ipa_endpoint_default_route_set() argument
1641 reg = ipa_reg(ipa, ROUTE); in ipa_endpoint_default_route_set()
1649 iowrite32(val, ipa->reg_virt + reg_offset(reg)); in ipa_endpoint_default_route_set()
1652 void ipa_endpoint_default_route_clear(struct ipa *ipa) in ipa_endpoint_default_route_clear() argument
1654 ipa_endpoint_default_route_set(ipa, 0); in ipa_endpoint_default_route_clear()
1658 * ipa_endpoint_reset_rx_aggr() - Reset RX endpoint with aggregation active
1659 * @endpoint: Endpoint to be reset
1663 * taken to ensure the IPA pipeline is properly cleared.
1669 struct device *dev = &endpoint->ipa->pdev->dev; in ipa_endpoint_reset_rx_aggr()
1670 struct ipa *ipa = endpoint->ipa; in ipa_endpoint_reset_rx_aggr() local
1671 struct gsi *gsi = &ipa->gsi; in ipa_endpoint_reset_rx_aggr()
1681 return -ENOMEM; in ipa_endpoint_reset_rx_aggr()
1685 ret = -ENOMEM; in ipa_endpoint_reset_rx_aggr()
1694 * active. We'll re-enable the doorbell (if appropriate) when in ipa_endpoint_reset_rx_aggr()
1697 gsi_channel_reset(gsi, endpoint->channel_id, false); in ipa_endpoint_reset_rx_aggr()
1703 ret = gsi_channel_start(gsi, endpoint->channel_id); in ipa_endpoint_reset_rx_aggr()
1707 ret = gsi_trans_read_byte(gsi, endpoint->channel_id, addr); in ipa_endpoint_reset_rx_aggr()
1711 /* Wait for aggregation to be closed on the channel */ in ipa_endpoint_reset_rx_aggr()
1717 } while (retries--); in ipa_endpoint_reset_rx_aggr()
1722 endpoint->endpoint_id); in ipa_endpoint_reset_rx_aggr()
1724 gsi_trans_read_byte_done(gsi, endpoint->channel_id); in ipa_endpoint_reset_rx_aggr()
1726 ret = gsi_channel_stop(gsi, endpoint->channel_id); in ipa_endpoint_reset_rx_aggr()
1730 /* Finally, reset and reconfigure the channel again (re-enabling in ipa_endpoint_reset_rx_aggr()
1731 * the doorbell engine if appropriate). Sleep for 1 millisecond to in ipa_endpoint_reset_rx_aggr()
1735 gsi_channel_reset(gsi, endpoint->channel_id, true); in ipa_endpoint_reset_rx_aggr()
1742 (void)gsi_channel_stop(gsi, endpoint->channel_id); in ipa_endpoint_reset_rx_aggr()
1755 u32 channel_id = endpoint->channel_id; in ipa_endpoint_reset()
1756 struct ipa *ipa = endpoint->ipa; in ipa_endpoint_reset() local
1760 /* On IPA v3.5.1, if an RX endpoint is reset while aggregation in ipa_endpoint_reset()
1761 * is active, we need to handle things specially to recover. in ipa_endpoint_reset()
1762 * All other cases just need to reset the underlying GSI channel. in ipa_endpoint_reset()
1764 special = ipa->version < IPA_VERSION_4_0 && !endpoint->toward_ipa && in ipa_endpoint_reset()
1765 endpoint->config.aggregation; in ipa_endpoint_reset()
1769 gsi_channel_reset(&ipa->gsi, channel_id, true); in ipa_endpoint_reset()
1772 dev_err(&ipa->pdev->dev, in ipa_endpoint_reset()
1774 ret, endpoint->channel_id, endpoint->endpoint_id); in ipa_endpoint_reset()
1779 if (endpoint->toward_ipa) { in ipa_endpoint_program()
1780 /* Newer versions of IPA use GSI channel flow control in ipa_endpoint_program()
1781 * instead of endpoint DELAY mode to prevent sending data. in ipa_endpoint_program()
1782 * Flow control is disabled for newly-allocated channels, in ipa_endpoint_program()
1784 * for AP TX channels. in ipa_endpoint_program()
1786 if (endpoint->ipa->version < IPA_VERSION_4_2) in ipa_endpoint_program()
1789 /* Ensure suspend mode is off on all AP RX endpoints */ in ipa_endpoint_program()
1799 if (!endpoint->toward_ipa) { in ipa_endpoint_program()
1800 if (endpoint->config.rx.holb_drop) in ipa_endpoint_program()
1813 u32 endpoint_id = endpoint->endpoint_id; in ipa_endpoint_enable_one()
1814 struct ipa *ipa = endpoint->ipa; in ipa_endpoint_enable_one() local
1815 struct gsi *gsi = &ipa->gsi; in ipa_endpoint_enable_one()
1818 ret = gsi_channel_start(gsi, endpoint->channel_id); in ipa_endpoint_enable_one()
1820 dev_err(&ipa->pdev->dev, in ipa_endpoint_enable_one()
1822 ret, endpoint->toward_ipa ? 'T' : 'R', in ipa_endpoint_enable_one()
1823 endpoint->channel_id, endpoint_id); in ipa_endpoint_enable_one()
1827 if (!endpoint->toward_ipa) { in ipa_endpoint_enable_one()
1828 ipa_interrupt_suspend_enable(ipa->interrupt, endpoint_id); in ipa_endpoint_enable_one()
1832 __set_bit(endpoint_id, ipa->enabled); in ipa_endpoint_enable_one()
1839 u32 endpoint_id = endpoint->endpoint_id; in ipa_endpoint_disable_one()
1840 struct ipa *ipa = endpoint->ipa; in ipa_endpoint_disable_one() local
1841 struct gsi *gsi = &ipa->gsi; in ipa_endpoint_disable_one()
1844 if (!test_bit(endpoint_id, ipa->enabled)) in ipa_endpoint_disable_one()
1847 __clear_bit(endpoint_id, endpoint->ipa->enabled); in ipa_endpoint_disable_one()
1849 if (!endpoint->toward_ipa) { in ipa_endpoint_disable_one()
1851 ipa_interrupt_suspend_disable(ipa->interrupt, endpoint_id); in ipa_endpoint_disable_one()
1854 /* Note that if stop fails, the channel's state is not well-defined */ in ipa_endpoint_disable_one()
1855 ret = gsi_channel_stop(gsi, endpoint->channel_id); in ipa_endpoint_disable_one()
1857 dev_err(&ipa->pdev->dev, in ipa_endpoint_disable_one()
1858 "error %d attempting to stop endpoint %u\n", ret, in ipa_endpoint_disable_one()
1864 struct device *dev = &endpoint->ipa->pdev->dev; in ipa_endpoint_suspend_one()
1865 struct gsi *gsi = &endpoint->ipa->gsi; in ipa_endpoint_suspend_one()
1868 if (!test_bit(endpoint->endpoint_id, endpoint->ipa->enabled)) in ipa_endpoint_suspend_one()
1871 if (!endpoint->toward_ipa) { in ipa_endpoint_suspend_one()
1876 ret = gsi_channel_suspend(gsi, endpoint->channel_id); in ipa_endpoint_suspend_one()
1879 endpoint->channel_id); in ipa_endpoint_suspend_one()
1884 struct device *dev = &endpoint->ipa->pdev->dev; in ipa_endpoint_resume_one()
1885 struct gsi *gsi = &endpoint->ipa->gsi; in ipa_endpoint_resume_one()
1888 if (!test_bit(endpoint->endpoint_id, endpoint->ipa->enabled)) in ipa_endpoint_resume_one()
1891 if (!endpoint->toward_ipa) in ipa_endpoint_resume_one()
1894 ret = gsi_channel_resume(gsi, endpoint->channel_id); in ipa_endpoint_resume_one()
1897 endpoint->channel_id); in ipa_endpoint_resume_one()
1898 else if (!endpoint->toward_ipa) in ipa_endpoint_resume_one()
1902 void ipa_endpoint_suspend(struct ipa *ipa) in ipa_endpoint_suspend() argument
1904 if (!ipa->setup_complete) in ipa_endpoint_suspend()
1907 if (ipa->modem_netdev) in ipa_endpoint_suspend()
1908 ipa_modem_suspend(ipa->modem_netdev); in ipa_endpoint_suspend()
1910 ipa_endpoint_suspend_one(ipa->name_map[IPA_ENDPOINT_AP_LAN_RX]); in ipa_endpoint_suspend()
1911 ipa_endpoint_suspend_one(ipa->name_map[IPA_ENDPOINT_AP_COMMAND_TX]); in ipa_endpoint_suspend()
1914 void ipa_endpoint_resume(struct ipa *ipa) in ipa_endpoint_resume() argument
1916 if (!ipa->setup_complete) in ipa_endpoint_resume()
1919 ipa_endpoint_resume_one(ipa->name_map[IPA_ENDPOINT_AP_COMMAND_TX]); in ipa_endpoint_resume()
1920 ipa_endpoint_resume_one(ipa->name_map[IPA_ENDPOINT_AP_LAN_RX]); in ipa_endpoint_resume()
1922 if (ipa->modem_netdev) in ipa_endpoint_resume()
1923 ipa_modem_resume(ipa->modem_netdev); in ipa_endpoint_resume()
1928 struct gsi *gsi = &endpoint->ipa->gsi; in ipa_endpoint_setup_one()
1929 u32 channel_id = endpoint->channel_id; in ipa_endpoint_setup_one()
1931 /* Only AP endpoints get set up */ in ipa_endpoint_setup_one()
1932 if (endpoint->ee_id != GSI_EE_AP) in ipa_endpoint_setup_one()
1935 endpoint->skb_frag_max = gsi->channel[channel_id].trans_tre_max - 1; in ipa_endpoint_setup_one()
1936 if (!endpoint->toward_ipa) { in ipa_endpoint_setup_one()
1940 clear_bit(IPA_REPLENISH_ENABLED, endpoint->replenish_flags); in ipa_endpoint_setup_one()
1941 clear_bit(IPA_REPLENISH_ACTIVE, endpoint->replenish_flags); in ipa_endpoint_setup_one()
1942 INIT_DELAYED_WORK(&endpoint->replenish_work, in ipa_endpoint_setup_one()
1948 __set_bit(endpoint->endpoint_id, endpoint->ipa->set_up); in ipa_endpoint_setup_one()
1953 __clear_bit(endpoint->endpoint_id, endpoint->ipa->set_up); in ipa_endpoint_teardown_one()
1955 if (!endpoint->toward_ipa) in ipa_endpoint_teardown_one()
1956 cancel_delayed_work_sync(&endpoint->replenish_work); in ipa_endpoint_teardown_one()
1961 void ipa_endpoint_setup(struct ipa *ipa) in ipa_endpoint_setup() argument
1965 for_each_set_bit(endpoint_id, ipa->defined, ipa->endpoint_count) in ipa_endpoint_setup()
1966 ipa_endpoint_setup_one(&ipa->endpoint[endpoint_id]); in ipa_endpoint_setup()
1969 void ipa_endpoint_teardown(struct ipa *ipa) in ipa_endpoint_teardown() argument
1973 for_each_set_bit(endpoint_id, ipa->set_up, ipa->endpoint_count) in ipa_endpoint_teardown()
1974 ipa_endpoint_teardown_one(&ipa->endpoint[endpoint_id]); in ipa_endpoint_teardown()
1977 void ipa_endpoint_deconfig(struct ipa *ipa) in ipa_endpoint_deconfig() argument
1979 ipa->available_count = 0; in ipa_endpoint_deconfig()
1980 bitmap_free(ipa->available); in ipa_endpoint_deconfig()
1981 ipa->available = NULL; in ipa_endpoint_deconfig()
1984 int ipa_endpoint_config(struct ipa *ipa) in ipa_endpoint_config() argument
1986 struct device *dev = &ipa->pdev->dev; in ipa_endpoint_config()
1996 /* Prior to IPA v3.5, the FLAVOR_0 register was not supported. in ipa_endpoint_config()
2003 * just set the available mask to support any endpoint, and in ipa_endpoint_config()
2006 if (ipa->version < IPA_VERSION_3_5) { in ipa_endpoint_config()
2007 ipa->available = bitmap_zalloc(IPA_ENDPOINT_MAX, GFP_KERNEL); in ipa_endpoint_config()
2008 if (!ipa->available) in ipa_endpoint_config()
2009 return -ENOMEM; in ipa_endpoint_config()
2010 ipa->available_count = IPA_ENDPOINT_MAX; in ipa_endpoint_config()
2012 bitmap_set(ipa->available, 0, IPA_ENDPOINT_MAX); in ipa_endpoint_config()
2020 reg = ipa_reg(ipa, FLAVOR_0); in ipa_endpoint_config()
2021 val = ioread32(ipa->reg_virt + reg_offset(reg)); in ipa_endpoint_config()
2023 /* Our RX is an IPA producer; our TX is an IPA consumer. */ in ipa_endpoint_config()
2032 return -EINVAL; in ipa_endpoint_config()
2035 /* Until IPA v5.0, the max endpoint ID was 32 */ in ipa_endpoint_config()
2036 hw_limit = ipa->version < IPA_VERSION_5_0 ? 32 : U8_MAX + 1; in ipa_endpoint_config()
2040 return -EINVAL; in ipa_endpoint_config()
2044 ipa->available = bitmap_zalloc(limit, GFP_KERNEL); in ipa_endpoint_config()
2045 if (!ipa->available) in ipa_endpoint_config()
2046 return -ENOMEM; in ipa_endpoint_config()
2047 ipa->available_count = limit; in ipa_endpoint_config()
2050 bitmap_set(ipa->available, 0, tx_count); in ipa_endpoint_config()
2051 bitmap_set(ipa->available, rx_base, rx_count); in ipa_endpoint_config()
2053 for_each_set_bit(endpoint_id, ipa->defined, ipa->endpoint_count) { in ipa_endpoint_config()
2058 endpoint_id, limit - 1); in ipa_endpoint_config()
2062 if (!test_bit(endpoint_id, ipa->available)) { in ipa_endpoint_config()
2069 endpoint = &ipa->endpoint[endpoint_id]; in ipa_endpoint_config()
2070 if (endpoint->toward_ipa) { in ipa_endpoint_config()
2084 ipa_endpoint_deconfig(ipa); in ipa_endpoint_config()
2086 return -EINVAL; in ipa_endpoint_config()
2089 static void ipa_endpoint_init_one(struct ipa *ipa, enum ipa_endpoint_name name, in ipa_endpoint_init_one() argument
2094 endpoint = &ipa->endpoint[data->endpoint_id]; in ipa_endpoint_init_one()
2096 if (data->ee_id == GSI_EE_AP) in ipa_endpoint_init_one()
2097 ipa->channel_map[data->channel_id] = endpoint; in ipa_endpoint_init_one()
2098 ipa->name_map[name] = endpoint; in ipa_endpoint_init_one()
2100 endpoint->ipa = ipa; in ipa_endpoint_init_one()
2101 endpoint->ee_id = data->ee_id; in ipa_endpoint_init_one()
2102 endpoint->channel_id = data->channel_id; in ipa_endpoint_init_one()
2103 endpoint->endpoint_id = data->endpoint_id; in ipa_endpoint_init_one()
2104 endpoint->toward_ipa = data->toward_ipa; in ipa_endpoint_init_one()
2105 endpoint->config = data->endpoint.config; in ipa_endpoint_init_one()
2107 __set_bit(endpoint->endpoint_id, ipa->defined); in ipa_endpoint_init_one()
2112 __clear_bit(endpoint->endpoint_id, endpoint->ipa->defined); in ipa_endpoint_exit_one()
2117 void ipa_endpoint_exit(struct ipa *ipa) in ipa_endpoint_exit() argument
2121 ipa->filtered = 0; in ipa_endpoint_exit()
2123 for_each_set_bit(endpoint_id, ipa->defined, ipa->endpoint_count) in ipa_endpoint_exit()
2124 ipa_endpoint_exit_one(&ipa->endpoint[endpoint_id]); in ipa_endpoint_exit()
2126 bitmap_free(ipa->enabled); in ipa_endpoint_exit()
2127 ipa->enabled = NULL; in ipa_endpoint_exit()
2128 bitmap_free(ipa->set_up); in ipa_endpoint_exit()
2129 ipa->set_up = NULL; in ipa_endpoint_exit()
2130 bitmap_free(ipa->defined); in ipa_endpoint_exit()
2131 ipa->defined = NULL; in ipa_endpoint_exit()
2133 memset(ipa->name_map, 0, sizeof(ipa->name_map)); in ipa_endpoint_exit()
2134 memset(ipa->channel_map, 0, sizeof(ipa->channel_map)); in ipa_endpoint_exit()
2138 int ipa_endpoint_init(struct ipa *ipa, u32 count, in ipa_endpoint_init() argument
2147 ipa->endpoint_count = ipa_endpoint_max(ipa, count, data) + 1; in ipa_endpoint_init()
2148 if (!ipa->endpoint_count) in ipa_endpoint_init()
2149 return -EINVAL; in ipa_endpoint_init()
2152 ipa->defined = bitmap_zalloc(ipa->endpoint_count, GFP_KERNEL); in ipa_endpoint_init()
2153 if (!ipa->defined) in ipa_endpoint_init()
2154 return -ENOMEM; in ipa_endpoint_init()
2156 ipa->set_up = bitmap_zalloc(ipa->endpoint_count, GFP_KERNEL); in ipa_endpoint_init()
2157 if (!ipa->set_up) in ipa_endpoint_init()
2160 ipa->enabled = bitmap_zalloc(ipa->endpoint_count, GFP_KERNEL); in ipa_endpoint_init()
2161 if (!ipa->enabled) in ipa_endpoint_init()
2169 ipa_endpoint_init_one(ipa, name, data); in ipa_endpoint_init()
2171 if (data->endpoint.filter_support) in ipa_endpoint_init()
2172 filtered |= BIT(data->endpoint_id); in ipa_endpoint_init()
2173 if (data->ee_id == GSI_EE_MODEM && data->toward_ipa) in ipa_endpoint_init()
2174 ipa->modem_tx_count++; in ipa_endpoint_init()
2178 if (!ipa_filtered_valid(ipa, filtered)) { in ipa_endpoint_init()
2179 ipa_endpoint_exit(ipa); in ipa_endpoint_init()
2181 return -EINVAL; in ipa_endpoint_init()
2184 ipa->filtered = filtered; in ipa_endpoint_init()
2189 bitmap_free(ipa->set_up); in ipa_endpoint_init()
2190 ipa->set_up = NULL; in ipa_endpoint_init()
2192 bitmap_free(ipa->defined); in ipa_endpoint_init()
2193 ipa->defined = NULL; in ipa_endpoint_init()
2195 return -ENOMEM; in ipa_endpoint_init()