1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Support for Intel Camera Imaging ISP subsystem. 4 * Copyright (c) 2010 - 2016, Intel Corporation. 5 * 6 * This program is free software; you can redistribute it and/or modify it 7 * under the terms and conditions of the GNU General Public License, 8 * version 2, as published by the Free Software Foundation. 9 * 10 * This program is distributed in the hope it will be useful, but WITHOUT 11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 13 * more details. 14 */ 15 16 #include "isp.h" 17 #include "vmem.h" 18 #include "vmem_local.h" 19 20 #if !defined(HRT_MEMORY_ACCESS) 21 #include "ia_css_device_access.h" 22 #endif 23 #include "assert_support.h" 24 25 typedef unsigned long long hive_uedge; 26 typedef hive_uedge *hive_wide; 27 28 /* Copied from SDK: sim_semantics.c */ 29 30 /* subword bits move like this: MSB[____xxxx____]LSB -> MSB[00000000xxxx]LSB */ 31 #define SUBWORD(w, start, end) (((w) & (((1ULL << ((end) - 1)) - 1) << 1 | 1)) >> (start)) 32 33 /* inverse subword bits move like this: MSB[xxxx____xxxx]LSB -> MSB[xxxx0000xxxx]LSB */ 34 #define INV_SUBWORD(w, start, end) ((w) & (~(((1ULL << ((end) - 1)) - 1) << 1 | 1) | ((1ULL << (start)) - 1))) 35 36 #define uedge_bits (8 * sizeof(hive_uedge)) 37 #define move_lower_bits(target, target_bit, src, src_bit) move_subword(target, target_bit, src, 0, src_bit) 38 #define move_upper_bits(target, target_bit, src, src_bit) move_subword(target, target_bit, src, src_bit, uedge_bits) 39 #define move_word(target, target_bit, src) move_subword(target, target_bit, src, 0, uedge_bits) 40 41 static void 42 move_subword( 43 hive_uedge *target, 44 unsigned int target_bit, 45 hive_uedge src, 46 unsigned int src_start, 47 unsigned int src_end) 48 { 49 unsigned int start_elem = target_bit / uedge_bits; 50 unsigned int start_bit = target_bit % uedge_bits; 51 unsigned int subword_width = src_end - src_start; 52 53 hive_uedge src_subword = SUBWORD(src, src_start, src_end); 54 55 if (subword_width + start_bit > uedge_bits) { /* overlap */ 56 hive_uedge old_val1; 57 hive_uedge old_val0 = INV_SUBWORD(target[start_elem], start_bit, uedge_bits); 58 59 target[start_elem] = old_val0 | (src_subword << start_bit); 60 old_val1 = INV_SUBWORD(target[start_elem + 1], 0, 61 subword_width + start_bit - uedge_bits); 62 target[start_elem + 1] = old_val1 | (src_subword >> (uedge_bits - start_bit)); 63 } else { 64 hive_uedge old_val = INV_SUBWORD(target[start_elem], start_bit, 65 start_bit + subword_width); 66 67 target[start_elem] = old_val | (src_subword << start_bit); 68 } 69 } 70 71 static void 72 hive_sim_wide_unpack( 73 hive_wide vector, 74 hive_wide elem, 75 hive_uint elem_bits, 76 hive_uint index) 77 { 78 /* pointers into wide_type: */ 79 unsigned int start_elem = (elem_bits * index) / uedge_bits; 80 unsigned int start_bit = (elem_bits * index) % uedge_bits; 81 unsigned int end_elem = (elem_bits * (index + 1) - 1) / uedge_bits; 82 unsigned int end_bit = ((elem_bits * (index + 1) - 1) % uedge_bits) + 1; 83 84 if (elem_bits == uedge_bits) { 85 /* easy case for speedup: */ 86 elem[0] = vector[index]; 87 } else if (start_elem == end_elem) { 88 /* only one (<=64 bits) element needs to be (partly) copied: */ 89 move_subword(elem, 0, vector[start_elem], start_bit, end_bit); 90 } else { 91 /* general case: handles edge spanning cases (includes >64bit elements) */ 92 unsigned int bits_written = 0; 93 unsigned int i; 94 95 move_upper_bits(elem, bits_written, vector[start_elem], start_bit); 96 bits_written += (64 - start_bit); 97 for (i = start_elem + 1; i < end_elem; i++) { 98 move_word(elem, bits_written, vector[i]); 99 bits_written += uedge_bits; 100 } 101 move_lower_bits(elem, bits_written, vector[end_elem], end_bit); 102 } 103 } 104 105 static void 106 hive_sim_wide_pack( 107 hive_wide vector, 108 hive_wide elem, 109 hive_uint elem_bits, 110 hive_uint index) 111 { 112 /* pointers into wide_type: */ 113 unsigned int start_elem = (elem_bits * index) / uedge_bits; 114 115 /* easy case for speedup: */ 116 if (elem_bits == uedge_bits) { 117 vector[start_elem] = elem[0]; 118 } else if (elem_bits > uedge_bits) { 119 unsigned int bits_to_write = elem_bits; 120 unsigned int start_bit = elem_bits * index; 121 unsigned int i = 0; 122 123 for (; bits_to_write > uedge_bits; 124 bits_to_write -= uedge_bits, i++, start_bit += uedge_bits) { 125 move_word(vector, start_bit, elem[i]); 126 } 127 move_lower_bits(vector, start_bit, elem[i], bits_to_write); 128 } else { 129 /* only one element needs to be (partly) copied: */ 130 move_lower_bits(vector, elem_bits * index, elem[0], elem_bits); 131 } 132 } 133 134 static void load_vector( 135 const isp_ID_t ID, 136 t_vmem_elem *to, 137 const t_vmem_elem *from) 138 { 139 unsigned int i; 140 hive_uedge *data; 141 unsigned int size = sizeof(short) * ISP_NWAY; 142 143 VMEM_ARRAY(v, 2 * ISP_NWAY); /* Need 2 vectors to work around vmem hss bug */ 144 assert(ISP_BAMEM_BASE[ID] != (hrt_address) - 1); 145 #if !defined(HRT_MEMORY_ACCESS) 146 ia_css_device_load(ISP_BAMEM_BASE[ID] + (unsigned long)from, &v[0][0], size); 147 #else 148 hrt_master_port_load(ISP_BAMEM_BASE[ID] + (unsigned long)from, &v[0][0], size); 149 #endif 150 data = (hive_uedge *)v; 151 for (i = 0; i < ISP_NWAY; i++) { 152 hive_uedge elem = 0; 153 154 hive_sim_wide_unpack(data, &elem, ISP_VEC_ELEMBITS, i); 155 to[i] = elem; 156 } 157 udelay(1); /* Spend at least 1 cycles per vector */ 158 } 159 160 static void store_vector( 161 const isp_ID_t ID, 162 t_vmem_elem *to, 163 const t_vmem_elem *from) 164 { 165 unsigned int i; 166 unsigned int size = sizeof(short) * ISP_NWAY; 167 168 VMEM_ARRAY(v, 2 * ISP_NWAY); /* Need 2 vectors to work around vmem hss bug */ 169 //load_vector (&v[1][0], &to[ISP_NWAY]); /* Fetch the next vector, since it will be overwritten. */ 170 hive_uedge *data = (hive_uedge *)v; 171 172 for (i = 0; i < ISP_NWAY; i++) { 173 hive_sim_wide_pack(data, (hive_wide)&from[i], ISP_VEC_ELEMBITS, i); 174 } 175 assert(ISP_BAMEM_BASE[ID] != (hrt_address) - 1); 176 #if !defined(HRT_MEMORY_ACCESS) 177 ia_css_device_store(ISP_BAMEM_BASE[ID] + (unsigned long)to, &v, size); 178 #else 179 //hrt_mem_store (ISP, VMEM, (unsigned)to, &v, siz); /* This will overwrite the next vector as well */ 180 hrt_master_port_store(ISP_BAMEM_BASE[ID] + (unsigned long)to, &v, size); 181 #endif 182 udelay(1); /* Spend at least 1 cycles per vector */ 183 } 184 185 void isp_vmem_load( 186 const isp_ID_t ID, 187 const t_vmem_elem *from, 188 t_vmem_elem *to, 189 unsigned int elems) /* In t_vmem_elem */ 190 { 191 unsigned int c; 192 const t_vmem_elem *vp = from; 193 194 assert(ID < N_ISP_ID); 195 assert((unsigned long)from % ISP_VEC_ALIGN == 0); 196 assert(elems % ISP_NWAY == 0); 197 for (c = 0; c < elems; c += ISP_NWAY) { 198 load_vector(ID, &to[c], vp); 199 vp = (t_vmem_elem *)((char *)vp + ISP_VEC_ALIGN); 200 } 201 } 202 203 void isp_vmem_store( 204 const isp_ID_t ID, 205 t_vmem_elem *to, 206 const t_vmem_elem *from, 207 unsigned int elems) /* In t_vmem_elem */ 208 { 209 unsigned int c; 210 t_vmem_elem *vp = to; 211 212 assert(ID < N_ISP_ID); 213 assert((unsigned long)to % ISP_VEC_ALIGN == 0); 214 assert(elems % ISP_NWAY == 0); 215 for (c = 0; c < elems; c += ISP_NWAY) { 216 store_vector(ID, vp, &from[c]); 217 vp = (t_vmem_elem *)((char *)vp + ISP_VEC_ALIGN); 218 } 219 } 220 221 void isp_vmem_2d_load( 222 const isp_ID_t ID, 223 const t_vmem_elem *from, 224 t_vmem_elem *to, 225 unsigned int height, 226 unsigned int width, 227 unsigned int stride_to, /* In t_vmem_elem */ 228 229 unsigned stride_from /* In t_vmem_elem */) 230 { 231 unsigned int h; 232 233 assert(ID < N_ISP_ID); 234 assert((unsigned long)from % ISP_VEC_ALIGN == 0); 235 assert(width % ISP_NWAY == 0); 236 assert(stride_from % ISP_NWAY == 0); 237 for (h = 0; h < height; h++) { 238 unsigned int c; 239 const t_vmem_elem *vp = from; 240 241 for (c = 0; c < width; c += ISP_NWAY) { 242 load_vector(ID, &to[stride_to * h + c], vp); 243 vp = (t_vmem_elem *)((char *)vp + ISP_VEC_ALIGN); 244 } 245 from = (const t_vmem_elem *)((const char *)from + stride_from / ISP_NWAY * 246 ISP_VEC_ALIGN); 247 } 248 } 249 250 void isp_vmem_2d_store( 251 const isp_ID_t ID, 252 t_vmem_elem *to, 253 const t_vmem_elem *from, 254 unsigned int height, 255 unsigned int width, 256 unsigned int stride_to, /* In t_vmem_elem */ 257 258 unsigned stride_from /* In t_vmem_elem */) 259 { 260 unsigned int h; 261 262 assert(ID < N_ISP_ID); 263 assert((unsigned long)to % ISP_VEC_ALIGN == 0); 264 assert(width % ISP_NWAY == 0); 265 assert(stride_to % ISP_NWAY == 0); 266 for (h = 0; h < height; h++) { 267 unsigned int c; 268 t_vmem_elem *vp = to; 269 270 for (c = 0; c < width; c += ISP_NWAY) { 271 store_vector(ID, vp, &from[stride_from * h + c]); 272 vp = (t_vmem_elem *)((char *)vp + ISP_VEC_ALIGN); 273 } 274 to = (t_vmem_elem *)((char *)to + stride_to / ISP_NWAY * ISP_VEC_ALIGN); 275 } 276 } 277