Loading...
1/* SPDX-License-Identifier: MIT */
2#ifndef __NVKM_FIFO_PRIV_H__
3#define __NVKM_FIFO_PRIV_H__
4#define nvkm_fifo(p) container_of((p), struct nvkm_fifo, engine)
5#include <engine/fifo.h>
6#include <core/enum.h>
7struct nvkm_cctx;
8struct nvkm_cgrp;
9struct nvkm_engn;
10struct nvkm_memory;
11struct nvkm_runl;
12struct nvkm_runq;
13struct nvkm_vctx;
14
15struct nvkm_fifo_func {
16 int (*chid_nr)(struct nvkm_fifo *);
17 int (*chid_ctor)(struct nvkm_fifo *, int nr);
18 int (*runq_nr)(struct nvkm_fifo *);
19 int (*runl_ctor)(struct nvkm_fifo *);
20
21 void (*init)(struct nvkm_fifo *);
22 void (*init_pbdmas)(struct nvkm_fifo *, u32 mask);
23
24 irqreturn_t (*intr)(struct nvkm_inth *);
25 void (*intr_mmu_fault_unit)(struct nvkm_fifo *, int unit);
26 void (*intr_ctxsw_timeout)(struct nvkm_fifo *, u32 engm);
27
28 const struct nvkm_fifo_func_mmu_fault {
29 void (*recover)(struct nvkm_fifo *, struct nvkm_fault_data *);
30 const struct nvkm_enum *access;
31 const struct nvkm_enum *engine;
32 const struct nvkm_enum *reason;
33 const struct nvkm_enum *hubclient;
34 const struct nvkm_enum *gpcclient;
35 } *mmu_fault;
36
37 void (*pause)(struct nvkm_fifo *, unsigned long *);
38 void (*start)(struct nvkm_fifo *, unsigned long *);
39
40 int (*nonstall_ctor)(struct nvkm_fifo *);
41 const struct nvkm_event_func *nonstall;
42
43 const struct nvkm_runl_func *runl;
44 const struct nvkm_runq_func *runq;
45 const struct nvkm_engn_func *engn;
46 const struct nvkm_engn_func *engn_sw;
47 const struct nvkm_engn_func *engn_ce;
48
49 struct nvkm_fifo_func_cgrp {
50 struct nvkm_sclass user;
51 const struct nvkm_cgrp_func *func;
52 bool force;
53 } cgrp;
54
55 struct nvkm_fifo_func_chan {
56 struct nvkm_sclass user;
57 const struct nvkm_chan_func *func;
58 } chan;
59};
60
61int nvkm_fifo_new_(const struct nvkm_fifo_func *, struct nvkm_device *, enum nvkm_subdev_type, int,
62 struct nvkm_fifo **);
63
64int nv04_fifo_chid_ctor(struct nvkm_fifo *, int);
65int nv04_fifo_runl_ctor(struct nvkm_fifo *);
66void nv04_fifo_init(struct nvkm_fifo *);
67irqreturn_t nv04_fifo_intr(struct nvkm_inth *);
68void nv04_fifo_pause(struct nvkm_fifo *, unsigned long *);
69void nv04_fifo_start(struct nvkm_fifo *, unsigned long *);
70extern const struct nvkm_runl_func nv04_runl;
71extern const struct nvkm_engn_func nv04_engn;
72extern const struct nvkm_cgrp_func nv04_cgrp;
73extern const struct nvkm_chan_func_inst nv04_chan_inst;
74extern const struct nvkm_chan_func_userd nv04_chan_userd;
75void nv04_chan_ramfc_clear(struct nvkm_chan *);
76void nv04_chan_start(struct nvkm_chan *);
77void nv04_chan_stop(struct nvkm_chan *);
78void nv04_eobj_ramht_del(struct nvkm_chan *, int);
79
80int nv10_fifo_chid_nr(struct nvkm_fifo *);
81
82int nv50_fifo_chid_nr(struct nvkm_fifo *);
83int nv50_fifo_chid_ctor(struct nvkm_fifo *, int);
84void nv50_fifo_init(struct nvkm_fifo *);
85extern const struct nvkm_runl_func nv50_runl;
86int nv50_runl_update(struct nvkm_runl *);
87int nv50_runl_wait(struct nvkm_runl *);
88extern const struct nvkm_engn_func nv50_engn_sw;
89extern const struct nvkm_chan_func_inst nv50_chan_inst;
90extern const struct nvkm_chan_func_userd nv50_chan_userd;
91void nv50_chan_unbind(struct nvkm_chan *);
92void nv50_chan_start(struct nvkm_chan *);
93void nv50_chan_stop(struct nvkm_chan *);
94void nv50_chan_preempt(struct nvkm_chan *);
95int nv50_eobj_ramht_add(struct nvkm_engn *, struct nvkm_object *, struct nvkm_chan *);
96void nv50_eobj_ramht_del(struct nvkm_chan *, int);
97
98extern const struct nvkm_event_func g84_fifo_nonstall;
99extern const struct nvkm_engn_func g84_engn;
100extern const struct nvkm_chan_func g84_chan;
101
102int gf100_fifo_chid_ctor(struct nvkm_fifo *, int);
103int gf100_fifo_runq_nr(struct nvkm_fifo *);
104bool gf100_fifo_intr_pbdma(struct nvkm_fifo *);
105void gf100_fifo_intr_mmu_fault(struct nvkm_fifo *);
106void gf100_fifo_intr_mmu_fault_unit(struct nvkm_fifo *, int);
107void gf100_fifo_intr_sched(struct nvkm_fifo *);
108void gf100_fifo_intr_ctxsw_timeout(struct nvkm_fifo *, u32);
109void gf100_fifo_mmu_fault_recover(struct nvkm_fifo *, struct nvkm_fault_data *);
110extern const struct nvkm_enum gf100_fifo_mmu_fault_access[];
111extern const struct nvkm_event_func gf100_fifo_nonstall;
112bool gf100_runl_preempt_pending(struct nvkm_runl *);
113void gf100_runq_init(struct nvkm_runq *);
114bool gf100_runq_intr(struct nvkm_runq *, struct nvkm_runl *);
115void gf100_engn_mmu_fault_trigger(struct nvkm_engn *);
116bool gf100_engn_mmu_fault_triggered(struct nvkm_engn *);
117extern const struct nvkm_engn_func gf100_engn_sw;
118extern const struct nvkm_chan_func_inst gf100_chan_inst;
119void gf100_chan_userd_clear(struct nvkm_chan *);
120void gf100_chan_preempt(struct nvkm_chan *);
121
122int gk104_fifo_chid_nr(struct nvkm_fifo *);
123int gk104_fifo_runl_ctor(struct nvkm_fifo *);
124void gk104_fifo_init(struct nvkm_fifo *);
125void gk104_fifo_init_pbdmas(struct nvkm_fifo *, u32);
126irqreturn_t gk104_fifo_intr(struct nvkm_inth *);
127void gk104_fifo_intr_runlist(struct nvkm_fifo *);
128void gk104_fifo_intr_chsw(struct nvkm_fifo *);
129void gk104_fifo_intr_bind(struct nvkm_fifo *);
130extern const struct nvkm_fifo_func_mmu_fault gk104_fifo_mmu_fault;
131extern const struct nvkm_enum gk104_fifo_mmu_fault_reason[];
132extern const struct nvkm_enum gk104_fifo_mmu_fault_hubclient[];
133extern const struct nvkm_enum gk104_fifo_mmu_fault_gpcclient[];
134void gk104_runl_insert_chan(struct nvkm_chan *, struct nvkm_memory *, u64);
135void gk104_runl_commit(struct nvkm_runl *, struct nvkm_memory *, u32, int);
136bool gk104_runl_pending(struct nvkm_runl *);
137void gk104_runl_block(struct nvkm_runl *, u32);
138void gk104_runl_allow(struct nvkm_runl *, u32);
139void gk104_runl_fault_clear(struct nvkm_runl *);
140extern const struct nvkm_runq_func gk104_runq;
141void gk104_runq_init(struct nvkm_runq *);
142bool gk104_runq_intr(struct nvkm_runq *, struct nvkm_runl *);
143extern const struct nvkm_bitfield gk104_runq_intr_0_names[];
144bool gk104_runq_idle(struct nvkm_runq *);
145extern const struct nvkm_engn_func gk104_engn;
146bool gk104_engn_chsw(struct nvkm_engn *);
147int gk104_engn_cxid(struct nvkm_engn *, bool *cgid);
148int gk104_ectx_ctor(struct nvkm_engn *, struct nvkm_vctx *);
149extern const struct nvkm_engn_func gk104_engn_ce;
150extern const struct nvkm_chan_func_userd gk104_chan_userd;
151extern const struct nvkm_chan_func_ramfc gk104_chan_ramfc;
152void gk104_chan_bind(struct nvkm_chan *);
153void gk104_chan_bind_inst(struct nvkm_chan *);
154void gk104_chan_unbind(struct nvkm_chan *);
155void gk104_chan_start(struct nvkm_chan *);
156void gk104_chan_stop(struct nvkm_chan *);
157
158int gk110_fifo_chid_ctor(struct nvkm_fifo *, int);
159extern const struct nvkm_runl_func gk110_runl;
160extern const struct nvkm_cgrp_func gk110_cgrp;
161void gk110_runl_insert_cgrp(struct nvkm_cgrp *, struct nvkm_memory *, u64);
162extern const struct nvkm_chan_func gk110_chan;
163void gk110_chan_preempt(struct nvkm_chan *);
164
165extern const struct nvkm_runq_func gk208_runq;
166void gk208_runq_init(struct nvkm_runq *);
167
168void gm107_fifo_intr_mmu_fault_unit(struct nvkm_fifo *, int);
169extern const struct nvkm_fifo_func_mmu_fault gm107_fifo_mmu_fault;
170extern const struct nvkm_runl_func gm107_runl;
171extern const struct nvkm_chan_func gm107_chan;
172
173int gm200_fifo_chid_nr(struct nvkm_fifo *);
174int gm200_fifo_runq_nr(struct nvkm_fifo *);
175
176extern const struct nvkm_enum gv100_fifo_mmu_fault_access[];
177extern const struct nvkm_enum gv100_fifo_mmu_fault_reason[];
178extern const struct nvkm_enum gv100_fifo_mmu_fault_hubclient[];
179extern const struct nvkm_enum gv100_fifo_mmu_fault_gpcclient[];
180void gv100_runl_insert_cgrp(struct nvkm_cgrp *, struct nvkm_memory *, u64);
181void gv100_runl_insert_chan(struct nvkm_chan *, struct nvkm_memory *, u64);
182void gv100_runl_preempt(struct nvkm_runl *);
183extern const struct nvkm_runq_func gv100_runq;
184extern const struct nvkm_engn_func gv100_engn;
185void gv100_ectx_bind(struct nvkm_engn *, struct nvkm_cctx *, struct nvkm_chan *);
186extern const struct nvkm_engn_func gv100_engn_ce;
187int gv100_ectx_ce_ctor(struct nvkm_engn *, struct nvkm_vctx *);
188void gv100_ectx_ce_bind(struct nvkm_engn *, struct nvkm_cctx *, struct nvkm_chan *);
189extern const struct nvkm_chan_func_userd gv100_chan_userd;
190extern const struct nvkm_chan_func_ramfc gv100_chan_ramfc;
191
192void tu102_fifo_intr_ctxsw_timeout_info(struct nvkm_engn *, u32 info);
193extern const struct nvkm_fifo_func_mmu_fault tu102_fifo_mmu_fault;
194
195int ga100_fifo_runl_ctor(struct nvkm_fifo *);
196int ga100_fifo_nonstall_ctor(struct nvkm_fifo *);
197extern const struct nvkm_event_func ga100_fifo_nonstall;
198extern const struct nvkm_runl_func ga100_runl;
199extern const struct nvkm_runq_func ga100_runq;
200extern const struct nvkm_engn_func ga100_engn;
201extern const struct nvkm_engn_func ga100_engn_ce;
202extern const struct nvkm_cgrp_func ga100_cgrp;
203extern const struct nvkm_chan_func ga100_chan;
204
205int nvkm_uchan_new(struct nvkm_fifo *, struct nvkm_cgrp *, const struct nvkm_oclass *,
206 void *argv, u32 argc, struct nvkm_object **);
207int nvkm_ucgrp_new(struct nvkm_fifo *, const struct nvkm_oclass *, void *argv, u32 argc,
208 struct nvkm_object **);
209#endif
1/* SPDX-License-Identifier: MIT */
2#ifndef __NVKM_FIFO_PRIV_H__
3#define __NVKM_FIFO_PRIV_H__
4#define nvkm_fifo(p) container_of((p), struct nvkm_fifo, engine)
5#include <engine/fifo.h>
6#include <core/enum.h>
7struct nvkm_cctx;
8struct nvkm_cgrp;
9struct nvkm_engn;
10struct nvkm_memory;
11struct nvkm_runl;
12struct nvkm_runq;
13struct nvkm_vctx;
14
15struct nvkm_fifo_func {
16 void (*dtor)(struct nvkm_fifo *);
17
18 int (*chid_nr)(struct nvkm_fifo *);
19 int (*chid_ctor)(struct nvkm_fifo *, int nr);
20 int (*runq_nr)(struct nvkm_fifo *);
21 int (*runl_ctor)(struct nvkm_fifo *);
22
23 void (*init)(struct nvkm_fifo *);
24 void (*init_pbdmas)(struct nvkm_fifo *, u32 mask);
25
26 irqreturn_t (*intr)(struct nvkm_inth *);
27 void (*intr_mmu_fault_unit)(struct nvkm_fifo *, int unit);
28 void (*intr_ctxsw_timeout)(struct nvkm_fifo *, u32 engm);
29
30 const struct nvkm_fifo_func_mmu_fault {
31 void (*recover)(struct nvkm_fifo *, struct nvkm_fault_data *);
32 const struct nvkm_enum *access;
33 const struct nvkm_enum *engine;
34 const struct nvkm_enum *reason;
35 const struct nvkm_enum *hubclient;
36 const struct nvkm_enum *gpcclient;
37 } *mmu_fault;
38
39 void (*pause)(struct nvkm_fifo *, unsigned long *);
40 void (*start)(struct nvkm_fifo *, unsigned long *);
41
42 int (*nonstall_ctor)(struct nvkm_fifo *);
43 const struct nvkm_event_func *nonstall;
44
45 const struct nvkm_runl_func *runl;
46 const struct nvkm_runq_func *runq;
47 const struct nvkm_engn_func *engn;
48 const struct nvkm_engn_func *engn_sw;
49 const struct nvkm_engn_func *engn_ce;
50
51 struct nvkm_fifo_func_cgrp {
52 struct nvkm_sclass user;
53 const struct nvkm_cgrp_func *func;
54 bool force;
55 } cgrp;
56
57 struct nvkm_fifo_func_chan {
58 struct nvkm_sclass user;
59 const struct nvkm_chan_func *func;
60 } chan;
61};
62
63int r535_fifo_new(const struct nvkm_fifo_func *, struct nvkm_device *, enum nvkm_subdev_type, int,
64 struct nvkm_fifo **);
65int nvkm_fifo_new_(const struct nvkm_fifo_func *, struct nvkm_device *, enum nvkm_subdev_type, int,
66 struct nvkm_fifo **);
67
68int nv04_fifo_chid_ctor(struct nvkm_fifo *, int);
69int nv04_fifo_runl_ctor(struct nvkm_fifo *);
70void nv04_fifo_init(struct nvkm_fifo *);
71irqreturn_t nv04_fifo_intr(struct nvkm_inth *);
72void nv04_fifo_pause(struct nvkm_fifo *, unsigned long *);
73void nv04_fifo_start(struct nvkm_fifo *, unsigned long *);
74extern const struct nvkm_runl_func nv04_runl;
75extern const struct nvkm_engn_func nv04_engn;
76extern const struct nvkm_cgrp_func nv04_cgrp;
77extern const struct nvkm_chan_func_inst nv04_chan_inst;
78extern const struct nvkm_chan_func_userd nv04_chan_userd;
79void nv04_chan_ramfc_clear(struct nvkm_chan *);
80void nv04_chan_start(struct nvkm_chan *);
81void nv04_chan_stop(struct nvkm_chan *);
82void nv04_eobj_ramht_del(struct nvkm_chan *, int);
83
84int nv10_fifo_chid_nr(struct nvkm_fifo *);
85
86int nv50_fifo_chid_nr(struct nvkm_fifo *);
87int nv50_fifo_chid_ctor(struct nvkm_fifo *, int);
88void nv50_fifo_init(struct nvkm_fifo *);
89extern const struct nvkm_runl_func nv50_runl;
90int nv50_runl_update(struct nvkm_runl *);
91int nv50_runl_wait(struct nvkm_runl *);
92extern const struct nvkm_engn_func nv50_engn_sw;
93extern const struct nvkm_chan_func_inst nv50_chan_inst;
94extern const struct nvkm_chan_func_userd nv50_chan_userd;
95void nv50_chan_unbind(struct nvkm_chan *);
96void nv50_chan_start(struct nvkm_chan *);
97void nv50_chan_stop(struct nvkm_chan *);
98void nv50_chan_preempt(struct nvkm_chan *);
99int nv50_eobj_ramht_add(struct nvkm_engn *, struct nvkm_object *, struct nvkm_chan *);
100void nv50_eobj_ramht_del(struct nvkm_chan *, int);
101
102extern const struct nvkm_event_func g84_fifo_nonstall;
103extern const struct nvkm_engn_func g84_engn;
104extern const struct nvkm_chan_func g84_chan;
105
106int gf100_fifo_chid_ctor(struct nvkm_fifo *, int);
107int gf100_fifo_runq_nr(struct nvkm_fifo *);
108bool gf100_fifo_intr_pbdma(struct nvkm_fifo *);
109void gf100_fifo_intr_mmu_fault(struct nvkm_fifo *);
110void gf100_fifo_intr_mmu_fault_unit(struct nvkm_fifo *, int);
111void gf100_fifo_intr_sched(struct nvkm_fifo *);
112void gf100_fifo_intr_ctxsw_timeout(struct nvkm_fifo *, u32);
113void gf100_fifo_mmu_fault_recover(struct nvkm_fifo *, struct nvkm_fault_data *);
114extern const struct nvkm_enum gf100_fifo_mmu_fault_access[];
115extern const struct nvkm_event_func gf100_fifo_nonstall;
116bool gf100_runl_preempt_pending(struct nvkm_runl *);
117void gf100_runq_init(struct nvkm_runq *);
118bool gf100_runq_intr(struct nvkm_runq *, struct nvkm_runl *);
119void gf100_engn_mmu_fault_trigger(struct nvkm_engn *);
120bool gf100_engn_mmu_fault_triggered(struct nvkm_engn *);
121extern const struct nvkm_engn_func gf100_engn_sw;
122extern const struct nvkm_chan_func_inst gf100_chan_inst;
123void gf100_chan_userd_clear(struct nvkm_chan *);
124void gf100_chan_preempt(struct nvkm_chan *);
125
126int gk104_fifo_chid_nr(struct nvkm_fifo *);
127int gk104_fifo_runl_ctor(struct nvkm_fifo *);
128void gk104_fifo_init(struct nvkm_fifo *);
129void gk104_fifo_init_pbdmas(struct nvkm_fifo *, u32);
130irqreturn_t gk104_fifo_intr(struct nvkm_inth *);
131void gk104_fifo_intr_runlist(struct nvkm_fifo *);
132void gk104_fifo_intr_chsw(struct nvkm_fifo *);
133void gk104_fifo_intr_bind(struct nvkm_fifo *);
134extern const struct nvkm_fifo_func_mmu_fault gk104_fifo_mmu_fault;
135extern const struct nvkm_enum gk104_fifo_mmu_fault_reason[];
136extern const struct nvkm_enum gk104_fifo_mmu_fault_hubclient[];
137extern const struct nvkm_enum gk104_fifo_mmu_fault_gpcclient[];
138void gk104_runl_insert_chan(struct nvkm_chan *, struct nvkm_memory *, u64);
139void gk104_runl_commit(struct nvkm_runl *, struct nvkm_memory *, u32, int);
140bool gk104_runl_pending(struct nvkm_runl *);
141void gk104_runl_block(struct nvkm_runl *, u32);
142void gk104_runl_allow(struct nvkm_runl *, u32);
143void gk104_runl_fault_clear(struct nvkm_runl *);
144extern const struct nvkm_runq_func gk104_runq;
145void gk104_runq_init(struct nvkm_runq *);
146bool gk104_runq_intr(struct nvkm_runq *, struct nvkm_runl *);
147extern const struct nvkm_bitfield gk104_runq_intr_0_names[];
148bool gk104_runq_idle(struct nvkm_runq *);
149extern const struct nvkm_engn_func gk104_engn;
150bool gk104_engn_chsw(struct nvkm_engn *);
151int gk104_engn_cxid(struct nvkm_engn *, bool *cgid);
152int gk104_ectx_ctor(struct nvkm_engn *, struct nvkm_vctx *);
153extern const struct nvkm_engn_func gk104_engn_ce;
154extern const struct nvkm_chan_func_userd gk104_chan_userd;
155extern const struct nvkm_chan_func_ramfc gk104_chan_ramfc;
156void gk104_chan_bind(struct nvkm_chan *);
157void gk104_chan_bind_inst(struct nvkm_chan *);
158void gk104_chan_unbind(struct nvkm_chan *);
159void gk104_chan_start(struct nvkm_chan *);
160void gk104_chan_stop(struct nvkm_chan *);
161
162int gk110_fifo_chid_ctor(struct nvkm_fifo *, int);
163extern const struct nvkm_runl_func gk110_runl;
164extern const struct nvkm_cgrp_func gk110_cgrp;
165void gk110_runl_insert_cgrp(struct nvkm_cgrp *, struct nvkm_memory *, u64);
166extern const struct nvkm_chan_func gk110_chan;
167void gk110_chan_preempt(struct nvkm_chan *);
168
169extern const struct nvkm_runq_func gk208_runq;
170void gk208_runq_init(struct nvkm_runq *);
171
172void gm107_fifo_intr_mmu_fault_unit(struct nvkm_fifo *, int);
173extern const struct nvkm_fifo_func_mmu_fault gm107_fifo_mmu_fault;
174extern const struct nvkm_runl_func gm107_runl;
175extern const struct nvkm_chan_func gm107_chan;
176
177int gm200_fifo_chid_nr(struct nvkm_fifo *);
178int gm200_fifo_runq_nr(struct nvkm_fifo *);
179
180extern const struct nvkm_enum gv100_fifo_mmu_fault_access[];
181extern const struct nvkm_enum gv100_fifo_mmu_fault_reason[];
182extern const struct nvkm_enum gv100_fifo_mmu_fault_hubclient[];
183extern const struct nvkm_enum gv100_fifo_mmu_fault_gpcclient[];
184void gv100_runl_insert_cgrp(struct nvkm_cgrp *, struct nvkm_memory *, u64);
185void gv100_runl_insert_chan(struct nvkm_chan *, struct nvkm_memory *, u64);
186void gv100_runl_preempt(struct nvkm_runl *);
187extern const struct nvkm_runq_func gv100_runq;
188extern const struct nvkm_engn_func gv100_engn;
189void gv100_ectx_bind(struct nvkm_engn *, struct nvkm_cctx *, struct nvkm_chan *);
190extern const struct nvkm_engn_func gv100_engn_ce;
191int gv100_ectx_ce_ctor(struct nvkm_engn *, struct nvkm_vctx *);
192void gv100_ectx_ce_bind(struct nvkm_engn *, struct nvkm_cctx *, struct nvkm_chan *);
193extern const struct nvkm_chan_func_userd gv100_chan_userd;
194extern const struct nvkm_chan_func_ramfc gv100_chan_ramfc;
195
196void tu102_fifo_intr_ctxsw_timeout_info(struct nvkm_engn *, u32 info);
197extern const struct nvkm_fifo_func_mmu_fault tu102_fifo_mmu_fault;
198
199int ga100_fifo_runl_ctor(struct nvkm_fifo *);
200int ga100_fifo_nonstall_ctor(struct nvkm_fifo *);
201extern const struct nvkm_event_func ga100_fifo_nonstall;
202extern const struct nvkm_runl_func ga100_runl;
203extern const struct nvkm_runq_func ga100_runq;
204extern const struct nvkm_engn_func ga100_engn;
205extern const struct nvkm_engn_func ga100_engn_ce;
206extern const struct nvkm_cgrp_func ga100_cgrp;
207extern const struct nvkm_chan_func ga100_chan;
208
209int nvkm_uchan_new(struct nvkm_fifo *, struct nvkm_cgrp *, const struct nvkm_oclass *,
210 void *argv, u32 argc, struct nvkm_object **);
211int nvkm_ucgrp_new(struct nvkm_fifo *, const struct nvkm_oclass *, void *argv, u32 argc,
212 struct nvkm_object **);
213#endif