1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2017-2018 The Linux Foundation. All rights reserved. */ 3 4 #include <linux/completion.h> 5 #include <linux/circ_buf.h> 6 #include <linux/list.h> 7 8 #include "a6xx_gmu.h" 9 #include "a6xx_gmu.xml.h" 10 #include "a6xx_gpu.h" 11 12 #define HFI_MSG_ID(val) [val] = #val 13 14 static const char * const a6xx_hfi_msg_id[] = { 15 HFI_MSG_ID(HFI_H2F_MSG_INIT), 16 HFI_MSG_ID(HFI_H2F_MSG_FW_VERSION), 17 HFI_MSG_ID(HFI_H2F_MSG_BW_TABLE), 18 HFI_MSG_ID(HFI_H2F_MSG_PERF_TABLE), 19 HFI_MSG_ID(HFI_H2F_MSG_TEST), 20 }; 21 22 static int a6xx_hfi_queue_read(struct a6xx_hfi_queue *queue, u32 *data, 23 u32 dwords) 24 { 25 struct a6xx_hfi_queue_header *header = queue->header; 26 u32 i, hdr, index = header->read_index; 27 28 if (header->read_index == header->write_index) { 29 header->rx_request = 1; 30 return 0; 31 } 32 33 hdr = queue->data[index]; 34 35 /* 36 * If we are to assume that the GMU firmware is in fact a rational actor 37 * and is programmed to not send us a larger response than we expect 38 * then we can also assume that if the header size is unexpectedly large 39 * that it is due to memory corruption and/or hardware failure. In this 40 * case the only reasonable course of action is to BUG() to help harden 41 * the failure. 42 */ 43 44 BUG_ON(HFI_HEADER_SIZE(hdr) > dwords); 45 46 for (i = 0; i < HFI_HEADER_SIZE(hdr); i++) { 47 data[i] = queue->data[index]; 48 index = (index + 1) % header->size; 49 } 50 51 header->read_index = index; 52 return HFI_HEADER_SIZE(hdr); 53 } 54 55 static int a6xx_hfi_queue_write(struct a6xx_gmu *gmu, 56 struct a6xx_hfi_queue *queue, u32 *data, u32 dwords) 57 { 58 struct a6xx_hfi_queue_header *header = queue->header; 59 u32 i, space, index = header->write_index; 60 61 spin_lock(&queue->lock); 62 63 space = CIRC_SPACE(header->write_index, header->read_index, 64 header->size); 65 if (space < dwords) { 66 header->dropped++; 67 spin_unlock(&queue->lock); 68 return -ENOSPC; 69 } 70 71 for (i = 0; i < dwords; i++) { 72 queue->data[index] = data[i]; 73 index = (index + 1) % header->size; 74 } 75 76 header->write_index = index; 77 spin_unlock(&queue->lock); 78 79 gmu_write(gmu, REG_A6XX_GMU_HOST2GMU_INTR_SET, 0x01); 80 return 0; 81 } 82 83 static int a6xx_hfi_wait_for_ack(struct a6xx_gmu *gmu, u32 id, u32 seqnum, 84 u32 *payload, u32 payload_size) 85 { 86 struct a6xx_hfi_queue *queue = &gmu->queues[HFI_RESPONSE_QUEUE]; 87 u32 val; 88 int ret; 89 90 /* Wait for a response */ 91 ret = gmu_poll_timeout(gmu, REG_A6XX_GMU_GMU2HOST_INTR_INFO, val, 92 val & A6XX_GMU_GMU2HOST_INTR_INFO_MSGQ, 100, 5000); 93 94 if (ret) { 95 DRM_DEV_ERROR(gmu->dev, 96 "Message %s id %d timed out waiting for response\n", 97 a6xx_hfi_msg_id[id], seqnum); 98 return -ETIMEDOUT; 99 } 100 101 /* Clear the interrupt */ 102 gmu_write(gmu, REG_A6XX_GMU_GMU2HOST_INTR_CLR, 103 A6XX_GMU_GMU2HOST_INTR_INFO_MSGQ); 104 105 for (;;) { 106 struct a6xx_hfi_msg_response resp; 107 108 /* Get the next packet */ 109 ret = a6xx_hfi_queue_read(queue, (u32 *) &resp, 110 sizeof(resp) >> 2); 111 112 /* If the queue is empty our response never made it */ 113 if (!ret) { 114 DRM_DEV_ERROR(gmu->dev, 115 "The HFI response queue is unexpectedly empty\n"); 116 117 return -ENOENT; 118 } 119 120 if (HFI_HEADER_ID(resp.header) == HFI_F2H_MSG_ERROR) { 121 struct a6xx_hfi_msg_error *error = 122 (struct a6xx_hfi_msg_error *) &resp; 123 124 DRM_DEV_ERROR(gmu->dev, "GMU firmware error %d\n", 125 error->code); 126 continue; 127 } 128 129 if (seqnum != HFI_HEADER_SEQNUM(resp.ret_header)) { 130 DRM_DEV_ERROR(gmu->dev, 131 "Unexpected message id %d on the response queue\n", 132 HFI_HEADER_SEQNUM(resp.ret_header)); 133 continue; 134 } 135 136 if (resp.error) { 137 DRM_DEV_ERROR(gmu->dev, 138 "Message %s id %d returned error %d\n", 139 a6xx_hfi_msg_id[id], seqnum, resp.error); 140 return -EINVAL; 141 } 142 143 /* All is well, copy over the buffer */ 144 if (payload && payload_size) 145 memcpy(payload, resp.payload, 146 min_t(u32, payload_size, sizeof(resp.payload))); 147 148 return 0; 149 } 150 } 151 152 static int a6xx_hfi_send_msg(struct a6xx_gmu *gmu, int id, 153 void *data, u32 size, u32 *payload, u32 payload_size) 154 { 155 struct a6xx_hfi_queue *queue = &gmu->queues[HFI_COMMAND_QUEUE]; 156 int ret, dwords = size >> 2; 157 u32 seqnum; 158 159 seqnum = atomic_inc_return(&queue->seqnum) % 0xfff; 160 161 /* First dword of the message is the message header - fill it in */ 162 *((u32 *) data) = (seqnum << 20) | (HFI_MSG_CMD << 16) | 163 (dwords << 8) | id; 164 165 ret = a6xx_hfi_queue_write(gmu, queue, data, dwords); 166 if (ret) { 167 DRM_DEV_ERROR(gmu->dev, "Unable to send message %s id %d\n", 168 a6xx_hfi_msg_id[id], seqnum); 169 return ret; 170 } 171 172 return a6xx_hfi_wait_for_ack(gmu, id, seqnum, payload, payload_size); 173 } 174 175 static int a6xx_hfi_send_gmu_init(struct a6xx_gmu *gmu, int boot_state) 176 { 177 struct a6xx_hfi_msg_gmu_init_cmd msg = { 0 }; 178 179 msg.dbg_buffer_addr = (u32) gmu->debug->iova; 180 msg.dbg_buffer_size = (u32) gmu->debug->size; 181 msg.boot_state = boot_state; 182 183 return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_INIT, &msg, sizeof(msg), 184 NULL, 0); 185 } 186 187 static int a6xx_hfi_get_fw_version(struct a6xx_gmu *gmu, u32 *version) 188 { 189 struct a6xx_hfi_msg_fw_version msg = { 0 }; 190 191 /* Currently supporting version 1.1 */ 192 msg.supported_version = (1 << 28) | (1 << 16); 193 194 return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_FW_VERSION, &msg, sizeof(msg), 195 version, sizeof(*version)); 196 } 197 198 static int a6xx_hfi_send_perf_table(struct a6xx_gmu *gmu) 199 { 200 struct a6xx_hfi_msg_perf_table msg = { 0 }; 201 int i; 202 203 msg.num_gpu_levels = gmu->nr_gpu_freqs; 204 msg.num_gmu_levels = gmu->nr_gmu_freqs; 205 206 for (i = 0; i < gmu->nr_gpu_freqs; i++) { 207 msg.gx_votes[i].vote = gmu->gx_arc_votes[i]; 208 msg.gx_votes[i].freq = gmu->gpu_freqs[i] / 1000; 209 } 210 211 for (i = 0; i < gmu->nr_gmu_freqs; i++) { 212 msg.cx_votes[i].vote = gmu->cx_arc_votes[i]; 213 msg.cx_votes[i].freq = gmu->gmu_freqs[i] / 1000; 214 } 215 216 return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_PERF_TABLE, &msg, sizeof(msg), 217 NULL, 0); 218 } 219 220 static void a618_build_bw_table(struct a6xx_hfi_msg_bw_table *msg) 221 { 222 /* Send a single "off" entry since the 618 GMU doesn't do bus scaling */ 223 msg->bw_level_num = 1; 224 225 msg->ddr_cmds_num = 3; 226 msg->ddr_wait_bitmask = 0x01; 227 228 msg->ddr_cmds_addrs[0] = 0x50000; 229 msg->ddr_cmds_addrs[1] = 0x5003c; 230 msg->ddr_cmds_addrs[2] = 0x5000c; 231 232 msg->ddr_cmds_data[0][0] = 0x40000000; 233 msg->ddr_cmds_data[0][1] = 0x40000000; 234 msg->ddr_cmds_data[0][2] = 0x40000000; 235 236 /* 237 * These are the CX (CNOC) votes - these are used by the GMU but the 238 * votes are known and fixed for the target 239 */ 240 msg->cnoc_cmds_num = 1; 241 msg->cnoc_wait_bitmask = 0x01; 242 243 msg->cnoc_cmds_addrs[0] = 0x5007c; 244 msg->cnoc_cmds_data[0][0] = 0x40000000; 245 msg->cnoc_cmds_data[1][0] = 0x60000001; 246 } 247 248 static void a6xx_build_bw_table(struct a6xx_hfi_msg_bw_table *msg) 249 { 250 /* Send a single "off" entry since the 630 GMU doesn't do bus scaling */ 251 msg->bw_level_num = 1; 252 253 msg->ddr_cmds_num = 3; 254 msg->ddr_wait_bitmask = 0x07; 255 256 msg->ddr_cmds_addrs[0] = 0x50000; 257 msg->ddr_cmds_addrs[1] = 0x5005c; 258 msg->ddr_cmds_addrs[2] = 0x5000c; 259 260 msg->ddr_cmds_data[0][0] = 0x40000000; 261 msg->ddr_cmds_data[0][1] = 0x40000000; 262 msg->ddr_cmds_data[0][2] = 0x40000000; 263 264 /* 265 * These are the CX (CNOC) votes. This is used but the values for the 266 * sdm845 GMU are known and fixed so we can hard code them. 267 */ 268 269 msg->cnoc_cmds_num = 3; 270 msg->cnoc_wait_bitmask = 0x05; 271 272 msg->cnoc_cmds_addrs[0] = 0x50034; 273 msg->cnoc_cmds_addrs[1] = 0x5007c; 274 msg->cnoc_cmds_addrs[2] = 0x5004c; 275 276 msg->cnoc_cmds_data[0][0] = 0x40000000; 277 msg->cnoc_cmds_data[0][1] = 0x00000000; 278 msg->cnoc_cmds_data[0][2] = 0x40000000; 279 280 msg->cnoc_cmds_data[1][0] = 0x60000001; 281 msg->cnoc_cmds_data[1][1] = 0x20000001; 282 msg->cnoc_cmds_data[1][2] = 0x60000001; 283 } 284 285 286 static int a6xx_hfi_send_bw_table(struct a6xx_gmu *gmu) 287 { 288 struct a6xx_hfi_msg_bw_table msg = { 0 }; 289 struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu); 290 struct adreno_gpu *adreno_gpu = &a6xx_gpu->base; 291 292 if (adreno_is_a618(adreno_gpu)) 293 a618_build_bw_table(&msg); 294 else 295 a6xx_build_bw_table(&msg); 296 297 return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_BW_TABLE, &msg, sizeof(msg), 298 NULL, 0); 299 } 300 301 static int a6xx_hfi_send_test(struct a6xx_gmu *gmu) 302 { 303 struct a6xx_hfi_msg_test msg = { 0 }; 304 305 return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_TEST, &msg, sizeof(msg), 306 NULL, 0); 307 } 308 309 int a6xx_hfi_start(struct a6xx_gmu *gmu, int boot_state) 310 { 311 int ret; 312 313 ret = a6xx_hfi_send_gmu_init(gmu, boot_state); 314 if (ret) 315 return ret; 316 317 ret = a6xx_hfi_get_fw_version(gmu, NULL); 318 if (ret) 319 return ret; 320 321 /* 322 * We have to get exchange version numbers per the sequence but at this 323 * point th kernel driver doesn't need to know the exact version of 324 * the GMU firmware 325 */ 326 327 ret = a6xx_hfi_send_perf_table(gmu); 328 if (ret) 329 return ret; 330 331 ret = a6xx_hfi_send_bw_table(gmu); 332 if (ret) 333 return ret; 334 335 /* 336 * Let the GMU know that there won't be any more HFI messages until next 337 * boot 338 */ 339 a6xx_hfi_send_test(gmu); 340 341 return 0; 342 } 343 344 void a6xx_hfi_stop(struct a6xx_gmu *gmu) 345 { 346 int i; 347 348 for (i = 0; i < ARRAY_SIZE(gmu->queues); i++) { 349 struct a6xx_hfi_queue *queue = &gmu->queues[i]; 350 351 if (!queue->header) 352 continue; 353 354 if (queue->header->read_index != queue->header->write_index) 355 DRM_DEV_ERROR(gmu->dev, "HFI queue %d is not empty\n", i); 356 357 queue->header->read_index = 0; 358 queue->header->write_index = 0; 359 } 360 } 361 362 static void a6xx_hfi_queue_init(struct a6xx_hfi_queue *queue, 363 struct a6xx_hfi_queue_header *header, void *virt, u64 iova, 364 u32 id) 365 { 366 spin_lock_init(&queue->lock); 367 queue->header = header; 368 queue->data = virt; 369 atomic_set(&queue->seqnum, 0); 370 371 /* Set up the shared memory header */ 372 header->iova = iova; 373 header->type = 10 << 8 | id; 374 header->status = 1; 375 header->size = SZ_4K >> 2; 376 header->msg_size = 0; 377 header->dropped = 0; 378 header->rx_watermark = 1; 379 header->tx_watermark = 1; 380 header->rx_request = 1; 381 header->tx_request = 0; 382 header->read_index = 0; 383 header->write_index = 0; 384 } 385 386 void a6xx_hfi_init(struct a6xx_gmu *gmu) 387 { 388 struct a6xx_gmu_bo *hfi = gmu->hfi; 389 struct a6xx_hfi_queue_table_header *table = hfi->virt; 390 struct a6xx_hfi_queue_header *headers = hfi->virt + sizeof(*table); 391 u64 offset; 392 int table_size; 393 394 /* 395 * The table size is the size of the table header plus all of the queue 396 * headers 397 */ 398 table_size = sizeof(*table); 399 table_size += (ARRAY_SIZE(gmu->queues) * 400 sizeof(struct a6xx_hfi_queue_header)); 401 402 table->version = 0; 403 table->size = table_size; 404 /* First queue header is located immediately after the table header */ 405 table->qhdr0_offset = sizeof(*table) >> 2; 406 table->qhdr_size = sizeof(struct a6xx_hfi_queue_header) >> 2; 407 table->num_queues = ARRAY_SIZE(gmu->queues); 408 table->active_queues = ARRAY_SIZE(gmu->queues); 409 410 /* Command queue */ 411 offset = SZ_4K; 412 a6xx_hfi_queue_init(&gmu->queues[0], &headers[0], hfi->virt + offset, 413 hfi->iova + offset, 0); 414 415 /* GMU response queue */ 416 offset += SZ_4K; 417 a6xx_hfi_queue_init(&gmu->queues[1], &headers[1], hfi->virt + offset, 418 hfi->iova + offset, 4); 419 } 420