14c921d4dSJack Steiner /* 24c921d4dSJack Steiner * Copyright (c) 2008 Silicon Graphics, Inc. All Rights Reserved. 34c921d4dSJack Steiner * 44c921d4dSJack Steiner * This program is free software; you can redistribute it and/or modify 54c921d4dSJack Steiner * it under the terms of the GNU Lesser General Public License as published by 64c921d4dSJack Steiner * the Free Software Foundation; either version 2.1 of the License, or 74c921d4dSJack Steiner * (at your option) any later version. 84c921d4dSJack Steiner * 94c921d4dSJack Steiner * This program is distributed in the hope that it will be useful, 104c921d4dSJack Steiner * but WITHOUT ANY WARRANTY; without even the implied warranty of 114c921d4dSJack Steiner * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 124c921d4dSJack Steiner * GNU Lesser General Public License for more details. 134c921d4dSJack Steiner * 144c921d4dSJack Steiner * You should have received a copy of the GNU Lesser General Public License 154c921d4dSJack Steiner * along with this program; if not, write to the Free Software 164c921d4dSJack Steiner * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 174c921d4dSJack Steiner */ 184c921d4dSJack Steiner 194c921d4dSJack Steiner #ifndef __GRU_INSTRUCTIONS_H__ 204c921d4dSJack Steiner #define __GRU_INSTRUCTIONS_H__ 214c921d4dSJack Steiner 22fe5bb6b0SJack Steiner extern int gru_check_status_proc(void *cb); 23fe5bb6b0SJack Steiner extern int gru_wait_proc(void *cb); 24fe5bb6b0SJack Steiner extern void gru_wait_abort_proc(void *cb); 25fe5bb6b0SJack Steiner 26fe5bb6b0SJack Steiner 274c921d4dSJack Steiner 284c921d4dSJack Steiner /* 294c921d4dSJack Steiner * Architecture dependent functions 304c921d4dSJack Steiner */ 314c921d4dSJack Steiner 32923f7f69SJack Steiner #if defined(CONFIG_IA64) 334c921d4dSJack Steiner #include <linux/compiler.h> 344c921d4dSJack Steiner #include <asm/intrinsics.h> 35fe5bb6b0SJack Steiner #define __flush_cache(p) ia64_fc((unsigned long)p) 364c921d4dSJack Steiner /* Use volatile on IA64 to ensure ordering via st4.rel */ 374c921d4dSJack Steiner #define gru_ordered_store_int(p, v) \ 384c921d4dSJack Steiner do { \ 394c921d4dSJack Steiner barrier(); \ 404c921d4dSJack Steiner *((volatile int *)(p)) = v; /* force st.rel */ \ 414c921d4dSJack Steiner } while (0) 42923f7f69SJack Steiner #elif defined(CONFIG_X86_64) 434c921d4dSJack Steiner #define __flush_cache(p) clflush(p) 444c921d4dSJack Steiner #define gru_ordered_store_int(p, v) \ 454c921d4dSJack Steiner do { \ 464c921d4dSJack Steiner barrier(); \ 474c921d4dSJack Steiner *(int *)p = v; \ 484c921d4dSJack Steiner } while (0) 494c921d4dSJack Steiner #else 504c921d4dSJack Steiner #error "Unsupported architecture" 514c921d4dSJack Steiner #endif 524c921d4dSJack Steiner 534c921d4dSJack Steiner /* 544c921d4dSJack Steiner * Control block status and exception codes 554c921d4dSJack Steiner */ 564c921d4dSJack Steiner #define CBS_IDLE 0 574c921d4dSJack Steiner #define CBS_EXCEPTION 1 584c921d4dSJack Steiner #define CBS_ACTIVE 2 594c921d4dSJack Steiner #define CBS_CALL_OS 3 604c921d4dSJack Steiner 614c921d4dSJack Steiner /* CB substatus bitmasks */ 624c921d4dSJack Steiner #define CBSS_MSG_QUEUE_MASK 7 634c921d4dSJack Steiner #define CBSS_IMPLICIT_ABORT_ACTIVE_MASK 8 644c921d4dSJack Steiner 654c921d4dSJack Steiner /* CB substatus message queue values (low 3 bits of substatus) */ 664c921d4dSJack Steiner #define CBSS_NO_ERROR 0 674c921d4dSJack Steiner #define CBSS_LB_OVERFLOWED 1 684c921d4dSJack Steiner #define CBSS_QLIMIT_REACHED 2 694c921d4dSJack Steiner #define CBSS_PAGE_OVERFLOW 3 704c921d4dSJack Steiner #define CBSS_AMO_NACKED 4 714c921d4dSJack Steiner #define CBSS_PUT_NACKED 5 724c921d4dSJack Steiner 734c921d4dSJack Steiner /* 744c921d4dSJack Steiner * Structure used to fetch exception detail for CBs that terminate with 754c921d4dSJack Steiner * CBS_EXCEPTION 764c921d4dSJack Steiner */ 774c921d4dSJack Steiner struct control_block_extended_exc_detail { 784c921d4dSJack Steiner unsigned long cb; 794c921d4dSJack Steiner int opc; 804c921d4dSJack Steiner int ecause; 814c921d4dSJack Steiner int exopc; 824c921d4dSJack Steiner long exceptdet0; 834c921d4dSJack Steiner int exceptdet1; 84cd1334f0SJack Steiner int cbrstate; 85cd1334f0SJack Steiner int cbrexecstatus; 864c921d4dSJack Steiner }; 874c921d4dSJack Steiner 884c921d4dSJack Steiner /* 894c921d4dSJack Steiner * Instruction formats 904c921d4dSJack Steiner */ 914c921d4dSJack Steiner 924c921d4dSJack Steiner /* 934c921d4dSJack Steiner * Generic instruction format. 944c921d4dSJack Steiner * This definition has precise bit field definitions. 954c921d4dSJack Steiner */ 964c921d4dSJack Steiner struct gru_instruction_bits { 974c921d4dSJack Steiner /* DW 0 - low */ 984c921d4dSJack Steiner unsigned int icmd: 1; 994c921d4dSJack Steiner unsigned char ima: 3; /* CB_DelRep, unmapped mode */ 1004c921d4dSJack Steiner unsigned char reserved0: 4; 1014c921d4dSJack Steiner unsigned int xtype: 3; 1024c921d4dSJack Steiner unsigned int iaa0: 2; 1034c921d4dSJack Steiner unsigned int iaa1: 2; 1044c921d4dSJack Steiner unsigned char reserved1: 1; 1054c921d4dSJack Steiner unsigned char opc: 8; /* opcode */ 1064c921d4dSJack Steiner unsigned char exopc: 8; /* extended opcode */ 1074c921d4dSJack Steiner /* DW 0 - high */ 1084c921d4dSJack Steiner unsigned int idef2: 22; /* TRi0 */ 1094c921d4dSJack Steiner unsigned char reserved2: 2; 1104c921d4dSJack Steiner unsigned char istatus: 2; 1114c921d4dSJack Steiner unsigned char isubstatus:4; 112cd1334f0SJack Steiner unsigned char reserved3: 1; 113cd1334f0SJack Steiner unsigned char tlb_fault_color: 1; 1144c921d4dSJack Steiner /* DW 1 */ 1154c921d4dSJack Steiner unsigned long idef4; /* 42 bits: TRi1, BufSize */ 1164c921d4dSJack Steiner /* DW 2-6 */ 1174c921d4dSJack Steiner unsigned long idef1; /* BAddr0 */ 1184c921d4dSJack Steiner unsigned long idef5; /* Nelem */ 1194c921d4dSJack Steiner unsigned long idef6; /* Stride, Operand1 */ 1204c921d4dSJack Steiner unsigned long idef3; /* BAddr1, Value, Operand2 */ 1214c921d4dSJack Steiner unsigned long reserved4; 1224c921d4dSJack Steiner /* DW 7 */ 1234c921d4dSJack Steiner unsigned long avalue; /* AValue */ 1244c921d4dSJack Steiner }; 1254c921d4dSJack Steiner 1264c921d4dSJack Steiner /* 1274c921d4dSJack Steiner * Generic instruction with friendlier names. This format is used 1284c921d4dSJack Steiner * for inline instructions. 1294c921d4dSJack Steiner */ 1304c921d4dSJack Steiner struct gru_instruction { 1314c921d4dSJack Steiner /* DW 0 */ 1324c921d4dSJack Steiner unsigned int op32; /* icmd,xtype,iaa0,ima,opc */ 1334c921d4dSJack Steiner unsigned int tri0; 1344c921d4dSJack Steiner unsigned long tri1_bufsize; /* DW 1 */ 1354c921d4dSJack Steiner unsigned long baddr0; /* DW 2 */ 1364c921d4dSJack Steiner unsigned long nelem; /* DW 3 */ 1374c921d4dSJack Steiner unsigned long op1_stride; /* DW 4 */ 1384c921d4dSJack Steiner unsigned long op2_value_baddr1; /* DW 5 */ 1394c921d4dSJack Steiner unsigned long reserved0; /* DW 6 */ 1404c921d4dSJack Steiner unsigned long avalue; /* DW 7 */ 1414c921d4dSJack Steiner }; 1424c921d4dSJack Steiner 1434c921d4dSJack Steiner /* Some shifts and masks for the low 32 bits of a GRU command */ 1444c921d4dSJack Steiner #define GRU_CB_ICMD_SHFT 0 1454c921d4dSJack Steiner #define GRU_CB_ICMD_MASK 0x1 1464c921d4dSJack Steiner #define GRU_CB_XTYPE_SHFT 8 1474c921d4dSJack Steiner #define GRU_CB_XTYPE_MASK 0x7 1484c921d4dSJack Steiner #define GRU_CB_IAA0_SHFT 11 1494c921d4dSJack Steiner #define GRU_CB_IAA0_MASK 0x3 1504c921d4dSJack Steiner #define GRU_CB_IAA1_SHFT 13 1514c921d4dSJack Steiner #define GRU_CB_IAA1_MASK 0x3 1524c921d4dSJack Steiner #define GRU_CB_IMA_SHFT 1 1534c921d4dSJack Steiner #define GRU_CB_IMA_MASK 0x3 1544c921d4dSJack Steiner #define GRU_CB_OPC_SHFT 16 1554c921d4dSJack Steiner #define GRU_CB_OPC_MASK 0xff 1564c921d4dSJack Steiner #define GRU_CB_EXOPC_SHFT 24 1574c921d4dSJack Steiner #define GRU_CB_EXOPC_MASK 0xff 1584c921d4dSJack Steiner 1594c921d4dSJack Steiner /* GRU instruction opcodes (opc field) */ 1604c921d4dSJack Steiner #define OP_NOP 0x00 1614c921d4dSJack Steiner #define OP_BCOPY 0x01 1624c921d4dSJack Steiner #define OP_VLOAD 0x02 1634c921d4dSJack Steiner #define OP_IVLOAD 0x03 1644c921d4dSJack Steiner #define OP_VSTORE 0x04 1654c921d4dSJack Steiner #define OP_IVSTORE 0x05 1664c921d4dSJack Steiner #define OP_VSET 0x06 1674c921d4dSJack Steiner #define OP_IVSET 0x07 1684c921d4dSJack Steiner #define OP_MESQ 0x08 1694c921d4dSJack Steiner #define OP_GAMXR 0x09 1704c921d4dSJack Steiner #define OP_GAMIR 0x0a 1714c921d4dSJack Steiner #define OP_GAMIRR 0x0b 1724c921d4dSJack Steiner #define OP_GAMER 0x0c 1734c921d4dSJack Steiner #define OP_GAMERR 0x0d 1744c921d4dSJack Steiner #define OP_BSTORE 0x0e 1754c921d4dSJack Steiner #define OP_VFLUSH 0x0f 1764c921d4dSJack Steiner 1774c921d4dSJack Steiner 1784c921d4dSJack Steiner /* Extended opcodes values (exopc field) */ 1794c921d4dSJack Steiner 1804c921d4dSJack Steiner /* GAMIR - AMOs with implicit operands */ 1814c921d4dSJack Steiner #define EOP_IR_FETCH 0x01 /* Plain fetch of memory */ 1824c921d4dSJack Steiner #define EOP_IR_CLR 0x02 /* Fetch and clear */ 1834c921d4dSJack Steiner #define EOP_IR_INC 0x05 /* Fetch and increment */ 1844c921d4dSJack Steiner #define EOP_IR_DEC 0x07 /* Fetch and decrement */ 1854c921d4dSJack Steiner #define EOP_IR_QCHK1 0x0d /* Queue check, 64 byte msg */ 1864c921d4dSJack Steiner #define EOP_IR_QCHK2 0x0e /* Queue check, 128 byte msg */ 1874c921d4dSJack Steiner 1884c921d4dSJack Steiner /* GAMIRR - Registered AMOs with implicit operands */ 1894c921d4dSJack Steiner #define EOP_IRR_FETCH 0x01 /* Registered fetch of memory */ 1904c921d4dSJack Steiner #define EOP_IRR_CLR 0x02 /* Registered fetch and clear */ 1914c921d4dSJack Steiner #define EOP_IRR_INC 0x05 /* Registered fetch and increment */ 1924c921d4dSJack Steiner #define EOP_IRR_DEC 0x07 /* Registered fetch and decrement */ 1934c921d4dSJack Steiner #define EOP_IRR_DECZ 0x0f /* Registered fetch and decrement, update on zero*/ 1944c921d4dSJack Steiner 1954c921d4dSJack Steiner /* GAMER - AMOs with explicit operands */ 1964c921d4dSJack Steiner #define EOP_ER_SWAP 0x00 /* Exchange argument and memory */ 1974c921d4dSJack Steiner #define EOP_ER_OR 0x01 /* Logical OR with memory */ 1984c921d4dSJack Steiner #define EOP_ER_AND 0x02 /* Logical AND with memory */ 1994c921d4dSJack Steiner #define EOP_ER_XOR 0x03 /* Logical XOR with memory */ 2004c921d4dSJack Steiner #define EOP_ER_ADD 0x04 /* Add value to memory */ 2014c921d4dSJack Steiner #define EOP_ER_CSWAP 0x08 /* Compare with operand2, write operand1 if match*/ 2024c921d4dSJack Steiner #define EOP_ER_CADD 0x0c /* Queue check, operand1*64 byte msg */ 2034c921d4dSJack Steiner 2044c921d4dSJack Steiner /* GAMERR - Registered AMOs with explicit operands */ 2054c921d4dSJack Steiner #define EOP_ERR_SWAP 0x00 /* Exchange argument and memory */ 2064c921d4dSJack Steiner #define EOP_ERR_OR 0x01 /* Logical OR with memory */ 2074c921d4dSJack Steiner #define EOP_ERR_AND 0x02 /* Logical AND with memory */ 2084c921d4dSJack Steiner #define EOP_ERR_XOR 0x03 /* Logical XOR with memory */ 2094c921d4dSJack Steiner #define EOP_ERR_ADD 0x04 /* Add value to memory */ 2104c921d4dSJack Steiner #define EOP_ERR_CSWAP 0x08 /* Compare with operand2, write operand1 if match*/ 2114c921d4dSJack Steiner #define EOP_ERR_EPOLL 0x09 /* Poll for equality */ 2124c921d4dSJack Steiner #define EOP_ERR_NPOLL 0x0a /* Poll for inequality */ 2134c921d4dSJack Steiner 2144c921d4dSJack Steiner /* GAMXR - SGI Arithmetic unit */ 2154c921d4dSJack Steiner #define EOP_XR_CSWAP 0x0b /* Masked compare exchange */ 2164c921d4dSJack Steiner 2174c921d4dSJack Steiner 2184c921d4dSJack Steiner /* Transfer types (xtype field) */ 2194c921d4dSJack Steiner #define XTYPE_B 0x0 /* byte */ 2204c921d4dSJack Steiner #define XTYPE_S 0x1 /* short (2-byte) */ 2214c921d4dSJack Steiner #define XTYPE_W 0x2 /* word (4-byte) */ 2224c921d4dSJack Steiner #define XTYPE_DW 0x3 /* doubleword (8-byte) */ 2234c921d4dSJack Steiner #define XTYPE_CL 0x6 /* cacheline (64-byte) */ 2244c921d4dSJack Steiner 2254c921d4dSJack Steiner 2264c921d4dSJack Steiner /* Instruction access attributes (iaa0, iaa1 fields) */ 2274c921d4dSJack Steiner #define IAA_RAM 0x0 /* normal cached RAM access */ 2284c921d4dSJack Steiner #define IAA_NCRAM 0x2 /* noncoherent RAM access */ 2294c921d4dSJack Steiner #define IAA_MMIO 0x1 /* noncoherent memory-mapped I/O space */ 2304c921d4dSJack Steiner #define IAA_REGISTER 0x3 /* memory-mapped registers, etc. */ 2314c921d4dSJack Steiner 2324c921d4dSJack Steiner 2334c921d4dSJack Steiner /* Instruction mode attributes (ima field) */ 2344c921d4dSJack Steiner #define IMA_MAPPED 0x0 /* Virtual mode */ 2354c921d4dSJack Steiner #define IMA_CB_DELAY 0x1 /* hold read responses until status changes */ 2364c921d4dSJack Steiner #define IMA_UNMAPPED 0x2 /* bypass the TLBs (OS only) */ 2374c921d4dSJack Steiner #define IMA_INTERRUPT 0x4 /* Interrupt when instruction completes */ 2384c921d4dSJack Steiner 2394c921d4dSJack Steiner /* CBE ecause bits */ 2404c921d4dSJack Steiner #define CBE_CAUSE_RI (1 << 0) 2414c921d4dSJack Steiner #define CBE_CAUSE_INVALID_INSTRUCTION (1 << 1) 2424c921d4dSJack Steiner #define CBE_CAUSE_UNMAPPED_MODE_FORBIDDEN (1 << 2) 2434c921d4dSJack Steiner #define CBE_CAUSE_PE_CHECK_DATA_ERROR (1 << 3) 2444c921d4dSJack Steiner #define CBE_CAUSE_IAA_GAA_MISMATCH (1 << 4) 2454c921d4dSJack Steiner #define CBE_CAUSE_DATA_SEGMENT_LIMIT_EXCEPTION (1 << 5) 2464c921d4dSJack Steiner #define CBE_CAUSE_OS_FATAL_TLB_FAULT (1 << 6) 2474c921d4dSJack Steiner #define CBE_CAUSE_EXECUTION_HW_ERROR (1 << 7) 2484c921d4dSJack Steiner #define CBE_CAUSE_TLBHW_ERROR (1 << 8) 2494c921d4dSJack Steiner #define CBE_CAUSE_RA_REQUEST_TIMEOUT (1 << 9) 2504c921d4dSJack Steiner #define CBE_CAUSE_HA_REQUEST_TIMEOUT (1 << 10) 2514c921d4dSJack Steiner #define CBE_CAUSE_RA_RESPONSE_FATAL (1 << 11) 2524c921d4dSJack Steiner #define CBE_CAUSE_RA_RESPONSE_NON_FATAL (1 << 12) 2534c921d4dSJack Steiner #define CBE_CAUSE_HA_RESPONSE_FATAL (1 << 13) 2544c921d4dSJack Steiner #define CBE_CAUSE_HA_RESPONSE_NON_FATAL (1 << 14) 2554c921d4dSJack Steiner #define CBE_CAUSE_ADDRESS_SPACE_DECODE_ERROR (1 << 15) 2564c921d4dSJack Steiner #define CBE_CAUSE_RESPONSE_DATA_ERROR (1 << 16) 2574c921d4dSJack Steiner #define CBE_CAUSE_PROTOCOL_STATE_DATA_ERROR (1 << 17) 2584c921d4dSJack Steiner 259cd1334f0SJack Steiner /* CBE cbrexecstatus bits */ 260cd1334f0SJack Steiner #define CBR_EXS_ABORT_OCC_BIT 0 261cd1334f0SJack Steiner #define CBR_EXS_INT_OCC_BIT 1 262cd1334f0SJack Steiner #define CBR_EXS_PENDING_BIT 2 263cd1334f0SJack Steiner #define CBR_EXS_QUEUED_BIT 3 264cd1334f0SJack Steiner #define CBR_EXS_TLBHW_BIT 4 265cd1334f0SJack Steiner #define CBR_EXS_EXCEPTION_BIT 5 266cd1334f0SJack Steiner 267cd1334f0SJack Steiner #define CBR_EXS_ABORT_OCC (1 << CBR_EXS_ABORT_OCC_BIT) 268cd1334f0SJack Steiner #define CBR_EXS_INT_OCC (1 << CBR_EXS_INT_OCC_BIT) 269cd1334f0SJack Steiner #define CBR_EXS_PENDING (1 << CBR_EXS_PENDING_BIT) 270cd1334f0SJack Steiner #define CBR_EXS_QUEUED (1 << CBR_EXS_QUEUED_BIT) 271cd1334f0SJack Steiner #define CBR_EXS_TLBHW (1 << CBR_EXS_TLBHW_BIT) 272cd1334f0SJack Steiner #define CBR_EXS_EXCEPTION (1 << CBR_EXS_EXCEPTION_BIT) 273cd1334f0SJack Steiner 2744c921d4dSJack Steiner /* 2754c921d4dSJack Steiner * Exceptions are retried for the following cases. If any OTHER bits are set 2764c921d4dSJack Steiner * in ecause, the exception is not retryable. 2774c921d4dSJack Steiner */ 2784c921d4dSJack Steiner #define EXCEPTION_RETRY_BITS (CBE_CAUSE_RESPONSE_DATA_ERROR | \ 2794c921d4dSJack Steiner CBE_CAUSE_RA_REQUEST_TIMEOUT | \ 2804c921d4dSJack Steiner CBE_CAUSE_TLBHW_ERROR | \ 2814c921d4dSJack Steiner CBE_CAUSE_HA_REQUEST_TIMEOUT) 2824c921d4dSJack Steiner 2834c921d4dSJack Steiner /* Message queue head structure */ 2844c921d4dSJack Steiner union gru_mesqhead { 2854c921d4dSJack Steiner unsigned long val; 2864c921d4dSJack Steiner struct { 2874c921d4dSJack Steiner unsigned int head; 2884c921d4dSJack Steiner unsigned int limit; 2894c921d4dSJack Steiner }; 2904c921d4dSJack Steiner }; 2914c921d4dSJack Steiner 2924c921d4dSJack Steiner 2934c921d4dSJack Steiner /* Generate the low word of a GRU instruction */ 2944c921d4dSJack Steiner static inline unsigned int 2954c921d4dSJack Steiner __opword(unsigned char opcode, unsigned char exopc, unsigned char xtype, 2964c921d4dSJack Steiner unsigned char iaa0, unsigned char iaa1, 2974c921d4dSJack Steiner unsigned char ima) 2984c921d4dSJack Steiner { 2994c921d4dSJack Steiner return (1 << GRU_CB_ICMD_SHFT) | 3004c921d4dSJack Steiner (iaa0 << GRU_CB_IAA0_SHFT) | 3014c921d4dSJack Steiner (iaa1 << GRU_CB_IAA1_SHFT) | 3024c921d4dSJack Steiner (ima << GRU_CB_IMA_SHFT) | 3034c921d4dSJack Steiner (xtype << GRU_CB_XTYPE_SHFT) | 3044c921d4dSJack Steiner (opcode << GRU_CB_OPC_SHFT) | 3054c921d4dSJack Steiner (exopc << GRU_CB_EXOPC_SHFT); 3064c921d4dSJack Steiner } 3074c921d4dSJack Steiner 3084c921d4dSJack Steiner /* 3094c921d4dSJack Steiner * Architecture specific intrinsics 3104c921d4dSJack Steiner */ 3114c921d4dSJack Steiner static inline void gru_flush_cache(void *p) 3124c921d4dSJack Steiner { 3134c921d4dSJack Steiner __flush_cache(p); 3144c921d4dSJack Steiner } 3154c921d4dSJack Steiner 3164c921d4dSJack Steiner /* 3174c921d4dSJack Steiner * Store the lower 32 bits of the command including the "start" bit. Then 3184c921d4dSJack Steiner * start the instruction executing. 3194c921d4dSJack Steiner */ 3204c921d4dSJack Steiner static inline void gru_start_instruction(struct gru_instruction *ins, int op32) 3214c921d4dSJack Steiner { 3224c921d4dSJack Steiner gru_ordered_store_int(ins, op32); 323923f7f69SJack Steiner gru_flush_cache(ins); 3244c921d4dSJack Steiner } 3254c921d4dSJack Steiner 3264c921d4dSJack Steiner 3274c921d4dSJack Steiner /* Convert "hints" to IMA */ 3284c921d4dSJack Steiner #define CB_IMA(h) ((h) | IMA_UNMAPPED) 3294c921d4dSJack Steiner 3304c921d4dSJack Steiner /* Convert data segment cache line index into TRI0 / TRI1 value */ 3314c921d4dSJack Steiner #define GRU_DINDEX(i) ((i) * GRU_CACHE_LINE_BYTES) 3324c921d4dSJack Steiner 3334c921d4dSJack Steiner /* Inline functions for GRU instructions. 3344c921d4dSJack Steiner * Note: 3354c921d4dSJack Steiner * - nelem and stride are in elements 3364c921d4dSJack Steiner * - tri0/tri1 is in bytes for the beginning of the data segment. 3374c921d4dSJack Steiner */ 3384c921d4dSJack Steiner static inline void gru_vload(void *cb, unsigned long mem_addr, 3394c921d4dSJack Steiner unsigned int tri0, unsigned char xtype, unsigned long nelem, 3404c921d4dSJack Steiner unsigned long stride, unsigned long hints) 3414c921d4dSJack Steiner { 3424c921d4dSJack Steiner struct gru_instruction *ins = (struct gru_instruction *)cb; 3434c921d4dSJack Steiner 3444c921d4dSJack Steiner ins->baddr0 = (long)mem_addr; 3454c921d4dSJack Steiner ins->nelem = nelem; 3464c921d4dSJack Steiner ins->tri0 = tri0; 3474c921d4dSJack Steiner ins->op1_stride = stride; 3484c921d4dSJack Steiner gru_start_instruction(ins, __opword(OP_VLOAD, 0, xtype, IAA_RAM, 0, 3494c921d4dSJack Steiner CB_IMA(hints))); 3504c921d4dSJack Steiner } 3514c921d4dSJack Steiner 3524c921d4dSJack Steiner static inline void gru_vstore(void *cb, unsigned long mem_addr, 3534c921d4dSJack Steiner unsigned int tri0, unsigned char xtype, unsigned long nelem, 3544c921d4dSJack Steiner unsigned long stride, unsigned long hints) 3554c921d4dSJack Steiner { 3564c921d4dSJack Steiner struct gru_instruction *ins = (void *)cb; 3574c921d4dSJack Steiner 3584c921d4dSJack Steiner ins->baddr0 = (long)mem_addr; 3594c921d4dSJack Steiner ins->nelem = nelem; 3604c921d4dSJack Steiner ins->tri0 = tri0; 3614c921d4dSJack Steiner ins->op1_stride = stride; 3624c921d4dSJack Steiner gru_start_instruction(ins, __opword(OP_VSTORE, 0, xtype, IAA_RAM, 0, 3634c921d4dSJack Steiner CB_IMA(hints))); 3644c921d4dSJack Steiner } 3654c921d4dSJack Steiner 3664c921d4dSJack Steiner static inline void gru_ivload(void *cb, unsigned long mem_addr, 3674c921d4dSJack Steiner unsigned int tri0, unsigned int tri1, unsigned char xtype, 3684c921d4dSJack Steiner unsigned long nelem, unsigned long hints) 3694c921d4dSJack Steiner { 3704c921d4dSJack Steiner struct gru_instruction *ins = (void *)cb; 3714c921d4dSJack Steiner 3724c921d4dSJack Steiner ins->baddr0 = (long)mem_addr; 3734c921d4dSJack Steiner ins->nelem = nelem; 3744c921d4dSJack Steiner ins->tri0 = tri0; 3754c921d4dSJack Steiner ins->tri1_bufsize = tri1; 3764c921d4dSJack Steiner gru_start_instruction(ins, __opword(OP_IVLOAD, 0, xtype, IAA_RAM, 0, 3774c921d4dSJack Steiner CB_IMA(hints))); 3784c921d4dSJack Steiner } 3794c921d4dSJack Steiner 3804c921d4dSJack Steiner static inline void gru_ivstore(void *cb, unsigned long mem_addr, 3814c921d4dSJack Steiner unsigned int tri0, unsigned int tri1, 3824c921d4dSJack Steiner unsigned char xtype, unsigned long nelem, unsigned long hints) 3834c921d4dSJack Steiner { 3844c921d4dSJack Steiner struct gru_instruction *ins = (void *)cb; 3854c921d4dSJack Steiner 3864c921d4dSJack Steiner ins->baddr0 = (long)mem_addr; 3874c921d4dSJack Steiner ins->nelem = nelem; 3884c921d4dSJack Steiner ins->tri0 = tri0; 3894c921d4dSJack Steiner ins->tri1_bufsize = tri1; 3904c921d4dSJack Steiner gru_start_instruction(ins, __opword(OP_IVSTORE, 0, xtype, IAA_RAM, 0, 3914c921d4dSJack Steiner CB_IMA(hints))); 3924c921d4dSJack Steiner } 3934c921d4dSJack Steiner 3944c921d4dSJack Steiner static inline void gru_vset(void *cb, unsigned long mem_addr, 3954c921d4dSJack Steiner unsigned long value, unsigned char xtype, unsigned long nelem, 3964c921d4dSJack Steiner unsigned long stride, unsigned long hints) 3974c921d4dSJack Steiner { 3984c921d4dSJack Steiner struct gru_instruction *ins = (void *)cb; 3994c921d4dSJack Steiner 4004c921d4dSJack Steiner ins->baddr0 = (long)mem_addr; 4014c921d4dSJack Steiner ins->op2_value_baddr1 = value; 4024c921d4dSJack Steiner ins->nelem = nelem; 4034c921d4dSJack Steiner ins->op1_stride = stride; 4044c921d4dSJack Steiner gru_start_instruction(ins, __opword(OP_VSET, 0, xtype, IAA_RAM, 0, 4054c921d4dSJack Steiner CB_IMA(hints))); 4064c921d4dSJack Steiner } 4074c921d4dSJack Steiner 4084c921d4dSJack Steiner static inline void gru_ivset(void *cb, unsigned long mem_addr, 4094c921d4dSJack Steiner unsigned int tri1, unsigned long value, unsigned char xtype, 4104c921d4dSJack Steiner unsigned long nelem, unsigned long hints) 4114c921d4dSJack Steiner { 4124c921d4dSJack Steiner struct gru_instruction *ins = (void *)cb; 4134c921d4dSJack Steiner 4144c921d4dSJack Steiner ins->baddr0 = (long)mem_addr; 4154c921d4dSJack Steiner ins->op2_value_baddr1 = value; 4164c921d4dSJack Steiner ins->nelem = nelem; 4174c921d4dSJack Steiner ins->tri1_bufsize = tri1; 4184c921d4dSJack Steiner gru_start_instruction(ins, __opword(OP_IVSET, 0, xtype, IAA_RAM, 0, 4194c921d4dSJack Steiner CB_IMA(hints))); 4204c921d4dSJack Steiner } 4214c921d4dSJack Steiner 4224c921d4dSJack Steiner static inline void gru_vflush(void *cb, unsigned long mem_addr, 4234c921d4dSJack Steiner unsigned long nelem, unsigned char xtype, unsigned long stride, 4244c921d4dSJack Steiner unsigned long hints) 4254c921d4dSJack Steiner { 4264c921d4dSJack Steiner struct gru_instruction *ins = (void *)cb; 4274c921d4dSJack Steiner 4284c921d4dSJack Steiner ins->baddr0 = (long)mem_addr; 4294c921d4dSJack Steiner ins->op1_stride = stride; 4304c921d4dSJack Steiner ins->nelem = nelem; 4314c921d4dSJack Steiner gru_start_instruction(ins, __opword(OP_VFLUSH, 0, xtype, IAA_RAM, 0, 4324c921d4dSJack Steiner CB_IMA(hints))); 4334c921d4dSJack Steiner } 4344c921d4dSJack Steiner 4354c921d4dSJack Steiner static inline void gru_nop(void *cb, int hints) 4364c921d4dSJack Steiner { 4374c921d4dSJack Steiner struct gru_instruction *ins = (void *)cb; 4384c921d4dSJack Steiner 4394c921d4dSJack Steiner gru_start_instruction(ins, __opword(OP_NOP, 0, 0, 0, 0, CB_IMA(hints))); 4404c921d4dSJack Steiner } 4414c921d4dSJack Steiner 4424c921d4dSJack Steiner 4434c921d4dSJack Steiner static inline void gru_bcopy(void *cb, const unsigned long src, 4444c921d4dSJack Steiner unsigned long dest, 4454c921d4dSJack Steiner unsigned int tri0, unsigned int xtype, unsigned long nelem, 4464c921d4dSJack Steiner unsigned int bufsize, unsigned long hints) 4474c921d4dSJack Steiner { 4484c921d4dSJack Steiner struct gru_instruction *ins = (void *)cb; 4494c921d4dSJack Steiner 4504c921d4dSJack Steiner ins->baddr0 = (long)src; 4514c921d4dSJack Steiner ins->op2_value_baddr1 = (long)dest; 4524c921d4dSJack Steiner ins->nelem = nelem; 4534c921d4dSJack Steiner ins->tri0 = tri0; 4544c921d4dSJack Steiner ins->tri1_bufsize = bufsize; 4554c921d4dSJack Steiner gru_start_instruction(ins, __opword(OP_BCOPY, 0, xtype, IAA_RAM, 4564c921d4dSJack Steiner IAA_RAM, CB_IMA(hints))); 4574c921d4dSJack Steiner } 4584c921d4dSJack Steiner 4594c921d4dSJack Steiner static inline void gru_bstore(void *cb, const unsigned long src, 4604c921d4dSJack Steiner unsigned long dest, unsigned int tri0, unsigned int xtype, 4614c921d4dSJack Steiner unsigned long nelem, unsigned long hints) 4624c921d4dSJack Steiner { 4634c921d4dSJack Steiner struct gru_instruction *ins = (void *)cb; 4644c921d4dSJack Steiner 4654c921d4dSJack Steiner ins->baddr0 = (long)src; 4664c921d4dSJack Steiner ins->op2_value_baddr1 = (long)dest; 4674c921d4dSJack Steiner ins->nelem = nelem; 4684c921d4dSJack Steiner ins->tri0 = tri0; 4694c921d4dSJack Steiner gru_start_instruction(ins, __opword(OP_BSTORE, 0, xtype, 0, IAA_RAM, 4704c921d4dSJack Steiner CB_IMA(hints))); 4714c921d4dSJack Steiner } 4724c921d4dSJack Steiner 4734c921d4dSJack Steiner static inline void gru_gamir(void *cb, int exopc, unsigned long src, 4744c921d4dSJack Steiner unsigned int xtype, unsigned long hints) 4754c921d4dSJack Steiner { 4764c921d4dSJack Steiner struct gru_instruction *ins = (void *)cb; 4774c921d4dSJack Steiner 4784c921d4dSJack Steiner ins->baddr0 = (long)src; 4794c921d4dSJack Steiner gru_start_instruction(ins, __opword(OP_GAMIR, exopc, xtype, IAA_RAM, 0, 4804c921d4dSJack Steiner CB_IMA(hints))); 4814c921d4dSJack Steiner } 4824c921d4dSJack Steiner 4834c921d4dSJack Steiner static inline void gru_gamirr(void *cb, int exopc, unsigned long src, 4844c921d4dSJack Steiner unsigned int xtype, unsigned long hints) 4854c921d4dSJack Steiner { 4864c921d4dSJack Steiner struct gru_instruction *ins = (void *)cb; 4874c921d4dSJack Steiner 4884c921d4dSJack Steiner ins->baddr0 = (long)src; 4894c921d4dSJack Steiner gru_start_instruction(ins, __opword(OP_GAMIRR, exopc, xtype, IAA_RAM, 0, 4904c921d4dSJack Steiner CB_IMA(hints))); 4914c921d4dSJack Steiner } 4924c921d4dSJack Steiner 4934c921d4dSJack Steiner static inline void gru_gamer(void *cb, int exopc, unsigned long src, 4944c921d4dSJack Steiner unsigned int xtype, 4954c921d4dSJack Steiner unsigned long operand1, unsigned long operand2, 4964c921d4dSJack Steiner unsigned long hints) 4974c921d4dSJack Steiner { 4984c921d4dSJack Steiner struct gru_instruction *ins = (void *)cb; 4994c921d4dSJack Steiner 5004c921d4dSJack Steiner ins->baddr0 = (long)src; 5014c921d4dSJack Steiner ins->op1_stride = operand1; 5024c921d4dSJack Steiner ins->op2_value_baddr1 = operand2; 5034c921d4dSJack Steiner gru_start_instruction(ins, __opword(OP_GAMER, exopc, xtype, IAA_RAM, 0, 5044c921d4dSJack Steiner CB_IMA(hints))); 5054c921d4dSJack Steiner } 5064c921d4dSJack Steiner 5074c921d4dSJack Steiner static inline void gru_gamerr(void *cb, int exopc, unsigned long src, 5084c921d4dSJack Steiner unsigned int xtype, unsigned long operand1, 5094c921d4dSJack Steiner unsigned long operand2, unsigned long hints) 5104c921d4dSJack Steiner { 5114c921d4dSJack Steiner struct gru_instruction *ins = (void *)cb; 5124c921d4dSJack Steiner 5134c921d4dSJack Steiner ins->baddr0 = (long)src; 5144c921d4dSJack Steiner ins->op1_stride = operand1; 5154c921d4dSJack Steiner ins->op2_value_baddr1 = operand2; 5164c921d4dSJack Steiner gru_start_instruction(ins, __opword(OP_GAMERR, exopc, xtype, IAA_RAM, 0, 5174c921d4dSJack Steiner CB_IMA(hints))); 5184c921d4dSJack Steiner } 5194c921d4dSJack Steiner 5204c921d4dSJack Steiner static inline void gru_gamxr(void *cb, unsigned long src, 5214c921d4dSJack Steiner unsigned int tri0, unsigned long hints) 5224c921d4dSJack Steiner { 5234c921d4dSJack Steiner struct gru_instruction *ins = (void *)cb; 5244c921d4dSJack Steiner 5254c921d4dSJack Steiner ins->baddr0 = (long)src; 5264c921d4dSJack Steiner ins->nelem = 4; 5274c921d4dSJack Steiner gru_start_instruction(ins, __opword(OP_GAMXR, EOP_XR_CSWAP, XTYPE_DW, 5284c921d4dSJack Steiner IAA_RAM, 0, CB_IMA(hints))); 5294c921d4dSJack Steiner } 5304c921d4dSJack Steiner 5314c921d4dSJack Steiner static inline void gru_mesq(void *cb, unsigned long queue, 5324c921d4dSJack Steiner unsigned long tri0, unsigned long nelem, 5334c921d4dSJack Steiner unsigned long hints) 5344c921d4dSJack Steiner { 5354c921d4dSJack Steiner struct gru_instruction *ins = (void *)cb; 5364c921d4dSJack Steiner 5374c921d4dSJack Steiner ins->baddr0 = (long)queue; 5384c921d4dSJack Steiner ins->nelem = nelem; 5394c921d4dSJack Steiner ins->tri0 = tri0; 5404c921d4dSJack Steiner gru_start_instruction(ins, __opword(OP_MESQ, 0, XTYPE_CL, IAA_RAM, 0, 5414c921d4dSJack Steiner CB_IMA(hints))); 5424c921d4dSJack Steiner } 5434c921d4dSJack Steiner 5444c921d4dSJack Steiner static inline unsigned long gru_get_amo_value(void *cb) 5454c921d4dSJack Steiner { 5464c921d4dSJack Steiner struct gru_instruction *ins = (void *)cb; 5474c921d4dSJack Steiner 5484c921d4dSJack Steiner return ins->avalue; 5494c921d4dSJack Steiner } 5504c921d4dSJack Steiner 5514c921d4dSJack Steiner static inline int gru_get_amo_value_head(void *cb) 5524c921d4dSJack Steiner { 5534c921d4dSJack Steiner struct gru_instruction *ins = (void *)cb; 5544c921d4dSJack Steiner 5554c921d4dSJack Steiner return ins->avalue & 0xffffffff; 5564c921d4dSJack Steiner } 5574c921d4dSJack Steiner 5584c921d4dSJack Steiner static inline int gru_get_amo_value_limit(void *cb) 5594c921d4dSJack Steiner { 5604c921d4dSJack Steiner struct gru_instruction *ins = (void *)cb; 5614c921d4dSJack Steiner 5624c921d4dSJack Steiner return ins->avalue >> 32; 5634c921d4dSJack Steiner } 5644c921d4dSJack Steiner 5654c921d4dSJack Steiner static inline union gru_mesqhead gru_mesq_head(int head, int limit) 5664c921d4dSJack Steiner { 5674c921d4dSJack Steiner union gru_mesqhead mqh; 5684c921d4dSJack Steiner 5694c921d4dSJack Steiner mqh.head = head; 5704c921d4dSJack Steiner mqh.limit = limit; 5714c921d4dSJack Steiner return mqh; 5724c921d4dSJack Steiner } 5734c921d4dSJack Steiner 5744c921d4dSJack Steiner /* 5754c921d4dSJack Steiner * Get struct control_block_extended_exc_detail for CB. 5764c921d4dSJack Steiner */ 5774c921d4dSJack Steiner extern int gru_get_cb_exception_detail(void *cb, 5784c921d4dSJack Steiner struct control_block_extended_exc_detail *excdet); 5794c921d4dSJack Steiner 5804c921d4dSJack Steiner #define GRU_EXC_STR_SIZE 256 5814c921d4dSJack Steiner 5824c921d4dSJack Steiner 5834c921d4dSJack Steiner /* 5844c921d4dSJack Steiner * Control block definition for checking status 5854c921d4dSJack Steiner */ 5864c921d4dSJack Steiner struct gru_control_block_status { 5874c921d4dSJack Steiner unsigned int icmd :1; 588fe5bb6b0SJack Steiner unsigned int ima :3; 589fe5bb6b0SJack Steiner unsigned int reserved0 :4; 590fe5bb6b0SJack Steiner unsigned int unused1 :24; 5914c921d4dSJack Steiner unsigned int unused2 :24; 5924c921d4dSJack Steiner unsigned int istatus :2; 5934c921d4dSJack Steiner unsigned int isubstatus :4; 594fe5bb6b0SJack Steiner unsigned int unused3 :2; 5954c921d4dSJack Steiner }; 5964c921d4dSJack Steiner 5974c921d4dSJack Steiner /* Get CB status */ 5984c921d4dSJack Steiner static inline int gru_get_cb_status(void *cb) 5994c921d4dSJack Steiner { 6004c921d4dSJack Steiner struct gru_control_block_status *cbs = (void *)cb; 6014c921d4dSJack Steiner 6024c921d4dSJack Steiner return cbs->istatus; 6034c921d4dSJack Steiner } 6044c921d4dSJack Steiner 6054c921d4dSJack Steiner /* Get CB message queue substatus */ 6064c921d4dSJack Steiner static inline int gru_get_cb_message_queue_substatus(void *cb) 6074c921d4dSJack Steiner { 6084c921d4dSJack Steiner struct gru_control_block_status *cbs = (void *)cb; 6094c921d4dSJack Steiner 6104c921d4dSJack Steiner return cbs->isubstatus & CBSS_MSG_QUEUE_MASK; 6114c921d4dSJack Steiner } 6124c921d4dSJack Steiner 6134c921d4dSJack Steiner /* Get CB substatus */ 6144c921d4dSJack Steiner static inline int gru_get_cb_substatus(void *cb) 6154c921d4dSJack Steiner { 6164c921d4dSJack Steiner struct gru_control_block_status *cbs = (void *)cb; 6174c921d4dSJack Steiner 6184c921d4dSJack Steiner return cbs->isubstatus; 6194c921d4dSJack Steiner } 6204c921d4dSJack Steiner 6214c921d4dSJack Steiner /* Check the status of a CB. If the CB is in UPM mode, call the 6224c921d4dSJack Steiner * OS to handle the UPM status. 6234c921d4dSJack Steiner * Returns the CB status field value (0 for normal completion) 6244c921d4dSJack Steiner */ 6254c921d4dSJack Steiner static inline int gru_check_status(void *cb) 6264c921d4dSJack Steiner { 6274c921d4dSJack Steiner struct gru_control_block_status *cbs = (void *)cb; 628923f7f69SJack Steiner int ret; 6294c921d4dSJack Steiner 630923f7f69SJack Steiner ret = cbs->istatus; 6314c921d4dSJack Steiner if (ret == CBS_CALL_OS) 6324c921d4dSJack Steiner ret = gru_check_status_proc(cb); 6334c921d4dSJack Steiner return ret; 6344c921d4dSJack Steiner } 6354c921d4dSJack Steiner 6364c921d4dSJack Steiner /* Wait for CB to complete. 6374c921d4dSJack Steiner * Returns the CB status field value (0 for normal completion) 6384c921d4dSJack Steiner */ 6394c921d4dSJack Steiner static inline int gru_wait(void *cb) 6404c921d4dSJack Steiner { 6414c921d4dSJack Steiner struct gru_control_block_status *cbs = (void *)cb; 642923f7f69SJack Steiner int ret = cbs->istatus; 6434c921d4dSJack Steiner 6444c921d4dSJack Steiner if (ret != CBS_IDLE) 6454c921d4dSJack Steiner ret = gru_wait_proc(cb); 6464c921d4dSJack Steiner return ret; 6474c921d4dSJack Steiner } 6484c921d4dSJack Steiner 6494c921d4dSJack Steiner /* Wait for CB to complete. Aborts program if error. (Note: error does NOT 6504c921d4dSJack Steiner * mean TLB mis - only fatal errors such as memory parity error or user 6514c921d4dSJack Steiner * bugs will cause termination. 6524c921d4dSJack Steiner */ 6534c921d4dSJack Steiner static inline void gru_wait_abort(void *cb) 6544c921d4dSJack Steiner { 6554c921d4dSJack Steiner struct gru_control_block_status *cbs = (void *)cb; 6564c921d4dSJack Steiner 6574c921d4dSJack Steiner if (cbs->istatus != CBS_IDLE) 6584c921d4dSJack Steiner gru_wait_abort_proc(cb); 6594c921d4dSJack Steiner } 6604c921d4dSJack Steiner 6614c921d4dSJack Steiner 6624c921d4dSJack Steiner /* 6634c921d4dSJack Steiner * Get a pointer to a control block 6644c921d4dSJack Steiner * gseg - GSeg address returned from gru_get_thread_gru_segment() 6654c921d4dSJack Steiner * index - index of desired CB 6664c921d4dSJack Steiner */ 6674c921d4dSJack Steiner static inline void *gru_get_cb_pointer(void *gseg, 6684c921d4dSJack Steiner int index) 6694c921d4dSJack Steiner { 6704c921d4dSJack Steiner return gseg + GRU_CB_BASE + index * GRU_HANDLE_STRIDE; 6714c921d4dSJack Steiner } 6724c921d4dSJack Steiner 6734c921d4dSJack Steiner /* 6744c921d4dSJack Steiner * Get a pointer to a cacheline in the data segment portion of a GSeg 6754c921d4dSJack Steiner * gseg - GSeg address returned from gru_get_thread_gru_segment() 6764c921d4dSJack Steiner * index - index of desired cache line 6774c921d4dSJack Steiner */ 6784c921d4dSJack Steiner static inline void *gru_get_data_pointer(void *gseg, int index) 6794c921d4dSJack Steiner { 6804c921d4dSJack Steiner return gseg + GRU_DS_BASE + index * GRU_CACHE_LINE_BYTES; 6814c921d4dSJack Steiner } 6824c921d4dSJack Steiner 6834c921d4dSJack Steiner /* 6844c921d4dSJack Steiner * Convert a vaddr into the tri index within the GSEG 6854c921d4dSJack Steiner * vaddr - virtual address of within gseg 6864c921d4dSJack Steiner */ 6874c921d4dSJack Steiner static inline int gru_get_tri(void *vaddr) 6884c921d4dSJack Steiner { 6894c921d4dSJack Steiner return ((unsigned long)vaddr & (GRU_GSEG_PAGESIZE - 1)) - GRU_DS_BASE; 6904c921d4dSJack Steiner } 6914c921d4dSJack Steiner #endif /* __GRU_INSTRUCTIONS_H__ */ 692