1 /* SPDX-License-Identifier: MIT */ 2 #ifndef __NVKM_FIFO_PRIV_H__ 3 #define __NVKM_FIFO_PRIV_H__ 4 #define nvkm_fifo(p) container_of((p), struct nvkm_fifo, engine) 5 #include <engine/fifo.h> 6 #include <core/enum.h> 7 struct nvkm_cgrp; 8 struct nvkm_engn; 9 struct nvkm_memory; 10 struct nvkm_runl; 11 struct nvkm_runq; 12 struct gk104_fifo; 13 14 struct nvkm_fifo_chan_oclass; 15 struct nvkm_fifo_func { 16 void *(*dtor)(struct nvkm_fifo *); 17 18 int (*oneinit)(struct nvkm_fifo *); 19 int (*chid_nr)(struct nvkm_fifo *); 20 int (*chid_ctor)(struct nvkm_fifo *, int nr); 21 int (*runq_nr)(struct nvkm_fifo *); 22 int (*runl_ctor)(struct nvkm_fifo *); 23 24 void (*init)(struct nvkm_fifo *); 25 void (*init_pbdmas)(struct nvkm_fifo *, u32 mask); 26 27 irqreturn_t (*intr)(struct nvkm_inth *); 28 void (*intr_mmu_fault_unit)(struct nvkm_fifo *, int unit); 29 void (*intr_ctxsw_timeout)(struct nvkm_fifo *, u32 engm); 30 31 const struct nvkm_fifo_func_mmu_fault { 32 void (*recover)(struct nvkm_fifo *, struct nvkm_fault_data *); 33 const struct nvkm_enum *access; 34 const struct nvkm_enum *engine; 35 const struct nvkm_enum *reason; 36 const struct nvkm_enum *hubclient; 37 const struct nvkm_enum *gpcclient; 38 } *mmu_fault; 39 40 int (*engine_id)(struct nvkm_fifo *, struct nvkm_engine *); 41 void (*pause)(struct nvkm_fifo *, unsigned long *); 42 void (*start)(struct nvkm_fifo *, unsigned long *); 43 44 const struct nvkm_event_func *nonstall; 45 46 const struct nvkm_runl_func *runl; 47 const struct nvkm_runq_func *runq; 48 const struct nvkm_engn_func *engn; 49 const struct nvkm_engn_func *engn_sw; 50 const struct nvkm_engn_func *engn_ce; 51 52 struct nvkm_fifo_func_cgrp { 53 struct nvkm_sclass user; 54 const struct nvkm_cgrp_func *func; 55 bool force; 56 } cgrp; 57 58 struct nvkm_fifo_func_chan { 59 struct nvkm_sclass user; 60 const struct nvkm_chan_func *func; 61 const struct nvkm_fifo_chan_oclass { 62 int (*ctor)(struct nvkm_fifo *, const struct nvkm_oclass *, 63 void *data, u32 size, struct nvkm_object **); 64 } *oclass; 65 int (*ctor)(struct gk104_fifo *, const struct nvkm_oclass *, void *, u32, 66 struct nvkm_object **); 67 } chan; 68 }; 69 70 int nvkm_fifo_ctor(const struct nvkm_fifo_func *, struct nvkm_device *, enum nvkm_subdev_type, int, 71 struct nvkm_fifo *); 72 73 int nv04_fifo_chid_ctor(struct nvkm_fifo *, int); 74 int nv04_fifo_runl_ctor(struct nvkm_fifo *); 75 void nv04_fifo_init(struct nvkm_fifo *); 76 irqreturn_t nv04_fifo_intr(struct nvkm_inth *); 77 int nv04_fifo_engine_id(struct nvkm_fifo *, struct nvkm_engine *); 78 void nv04_fifo_pause(struct nvkm_fifo *, unsigned long *); 79 void nv04_fifo_start(struct nvkm_fifo *, unsigned long *); 80 extern const struct nvkm_runl_func nv04_runl; 81 extern const struct nvkm_engn_func nv04_engn; 82 extern const struct nvkm_cgrp_func nv04_cgrp; 83 extern const struct nvkm_chan_func_inst nv04_chan_inst; 84 extern const struct nvkm_chan_func_userd nv04_chan_userd; 85 void nv04_chan_ramfc_clear(struct nvkm_chan *); 86 void nv04_chan_start(struct nvkm_chan *); 87 void nv04_chan_stop(struct nvkm_chan *); 88 89 int nv10_fifo_chid_nr(struct nvkm_fifo *); 90 91 int nv50_fifo_chid_nr(struct nvkm_fifo *); 92 int nv50_fifo_chid_ctor(struct nvkm_fifo *, int); 93 void nv50_fifo_init(struct nvkm_fifo *); 94 extern const struct nvkm_runl_func nv50_runl; 95 int nv50_runl_update(struct nvkm_runl *); 96 int nv50_runl_wait(struct nvkm_runl *); 97 extern const struct nvkm_engn_func nv50_engn_sw; 98 extern const struct nvkm_chan_func_inst nv50_chan_inst; 99 extern const struct nvkm_chan_func_userd nv50_chan_userd; 100 void nv50_chan_unbind(struct nvkm_chan *); 101 void nv50_chan_start(struct nvkm_chan *); 102 void nv50_chan_stop(struct nvkm_chan *); 103 void nv50_chan_preempt(struct nvkm_chan *); 104 105 extern const struct nvkm_event_func g84_fifo_nonstall; 106 extern const struct nvkm_engn_func g84_engn; 107 extern const struct nvkm_chan_func g84_chan; 108 109 int gf100_fifo_chid_ctor(struct nvkm_fifo *, int); 110 int gf100_fifo_runq_nr(struct nvkm_fifo *); 111 bool gf100_fifo_intr_pbdma(struct nvkm_fifo *); 112 void gf100_fifo_intr_mmu_fault(struct nvkm_fifo *); 113 void gf100_fifo_intr_mmu_fault_unit(struct nvkm_fifo *, int); 114 void gf100_fifo_intr_sched(struct nvkm_fifo *); 115 void gf100_fifo_intr_ctxsw_timeout(struct nvkm_fifo *, u32); 116 void gf100_fifo_mmu_fault_recover(struct nvkm_fifo *, struct nvkm_fault_data *); 117 extern const struct nvkm_enum gf100_fifo_mmu_fault_access[]; 118 extern const struct nvkm_event_func gf100_fifo_nonstall; 119 bool gf100_runl_preempt_pending(struct nvkm_runl *); 120 void gf100_runq_init(struct nvkm_runq *); 121 bool gf100_runq_intr(struct nvkm_runq *, struct nvkm_runl *); 122 void gf100_engn_mmu_fault_trigger(struct nvkm_engn *); 123 bool gf100_engn_mmu_fault_triggered(struct nvkm_engn *); 124 extern const struct nvkm_engn_func gf100_engn_sw; 125 extern const struct nvkm_chan_func_inst gf100_chan_inst; 126 void gf100_chan_userd_clear(struct nvkm_chan *); 127 void gf100_chan_preempt(struct nvkm_chan *); 128 129 int gk104_fifo_chid_nr(struct nvkm_fifo *); 130 int gk104_fifo_runl_ctor(struct nvkm_fifo *); 131 void gk104_fifo_init(struct nvkm_fifo *); 132 void gk104_fifo_init_pbdmas(struct nvkm_fifo *, u32); 133 irqreturn_t gk104_fifo_intr(struct nvkm_inth *); 134 void gk104_fifo_intr_runlist(struct nvkm_fifo *); 135 void gk104_fifo_intr_chsw(struct nvkm_fifo *); 136 void gk104_fifo_intr_bind(struct nvkm_fifo *); 137 extern const struct nvkm_fifo_func_mmu_fault gk104_fifo_mmu_fault; 138 extern const struct nvkm_enum gk104_fifo_mmu_fault_reason[]; 139 extern const struct nvkm_enum gk104_fifo_mmu_fault_hubclient[]; 140 extern const struct nvkm_enum gk104_fifo_mmu_fault_gpcclient[]; 141 int gk104_fifo_engine_id(struct nvkm_fifo *, struct nvkm_engine *); 142 void gk104_runl_insert_chan(struct nvkm_chan *, struct nvkm_memory *, u64); 143 void gk104_runl_commit(struct nvkm_runl *, struct nvkm_memory *, u32, int); 144 bool gk104_runl_pending(struct nvkm_runl *); 145 void gk104_runl_block(struct nvkm_runl *, u32); 146 void gk104_runl_allow(struct nvkm_runl *, u32); 147 void gk104_runl_fault_clear(struct nvkm_runl *); 148 extern const struct nvkm_runq_func gk104_runq; 149 void gk104_runq_init(struct nvkm_runq *); 150 bool gk104_runq_intr(struct nvkm_runq *, struct nvkm_runl *); 151 extern const struct nvkm_bitfield gk104_runq_intr_0_names[]; 152 bool gk104_runq_idle(struct nvkm_runq *); 153 extern const struct nvkm_engn_func gk104_engn; 154 bool gk104_engn_chsw(struct nvkm_engn *); 155 int gk104_engn_cxid(struct nvkm_engn *, bool *cgid); 156 extern const struct nvkm_engn_func gk104_engn_ce; 157 extern const struct nvkm_chan_func_userd gk104_chan_userd; 158 extern const struct nvkm_chan_func_ramfc gk104_chan_ramfc; 159 void gk104_chan_bind(struct nvkm_chan *); 160 void gk104_chan_bind_inst(struct nvkm_chan *); 161 void gk104_chan_unbind(struct nvkm_chan *); 162 void gk104_chan_start(struct nvkm_chan *); 163 void gk104_chan_stop(struct nvkm_chan *); 164 165 int gk110_fifo_chid_ctor(struct nvkm_fifo *, int); 166 extern const struct nvkm_runl_func gk110_runl; 167 extern const struct nvkm_cgrp_func gk110_cgrp; 168 void gk110_runl_insert_cgrp(struct nvkm_cgrp *, struct nvkm_memory *, u64); 169 extern const struct nvkm_chan_func gk110_chan; 170 void gk110_chan_preempt(struct nvkm_chan *); 171 172 extern const struct nvkm_runq_func gk208_runq; 173 void gk208_runq_init(struct nvkm_runq *); 174 175 void gm107_fifo_intr_mmu_fault_unit(struct nvkm_fifo *, int); 176 extern const struct nvkm_fifo_func_mmu_fault gm107_fifo_mmu_fault; 177 extern const struct nvkm_runl_func gm107_runl; 178 extern const struct nvkm_chan_func gm107_chan; 179 180 int gm200_fifo_chid_nr(struct nvkm_fifo *); 181 int gm200_fifo_runq_nr(struct nvkm_fifo *); 182 183 extern const struct nvkm_enum gv100_fifo_mmu_fault_access[]; 184 extern const struct nvkm_enum gv100_fifo_mmu_fault_reason[]; 185 extern const struct nvkm_enum gv100_fifo_mmu_fault_hubclient[]; 186 extern const struct nvkm_enum gv100_fifo_mmu_fault_gpcclient[]; 187 void gv100_runl_insert_cgrp(struct nvkm_cgrp *, struct nvkm_memory *, u64); 188 void gv100_runl_insert_chan(struct nvkm_chan *, struct nvkm_memory *, u64); 189 void gv100_runl_preempt(struct nvkm_runl *); 190 extern const struct nvkm_runq_func gv100_runq; 191 extern const struct nvkm_engn_func gv100_engn; 192 extern const struct nvkm_engn_func gv100_engn_ce; 193 extern const struct nvkm_chan_func_userd gv100_chan_userd; 194 extern const struct nvkm_chan_func_ramfc gv100_chan_ramfc; 195 196 void tu102_fifo_intr_ctxsw_timeout_info(struct nvkm_engn *, u32 info); 197 extern const struct nvkm_fifo_func_mmu_fault tu102_fifo_mmu_fault; 198 199 int nvkm_uchan_new(struct nvkm_fifo *, struct nvkm_cgrp *, const struct nvkm_oclass *, 200 void *argv, u32 argc, struct nvkm_object **); 201 #endif 202