1d87f36a0SRajneesh Bhardwaj /* SPDX-License-Identifier: GPL-2.0 OR MIT */
2ed6e6a34SBen Goz /*
3d87f36a0SRajneesh Bhardwaj * Copyright 2014-2022 Advanced Micro Devices, Inc.
4ed6e6a34SBen Goz *
5ed6e6a34SBen Goz * Permission is hereby granted, free of charge, to any person obtaining a
6ed6e6a34SBen Goz * copy of this software and associated documentation files (the "Software"),
7ed6e6a34SBen Goz * to deal in the Software without restriction, including without limitation
8ed6e6a34SBen Goz * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9ed6e6a34SBen Goz * and/or sell copies of the Software, and to permit persons to whom the
10ed6e6a34SBen Goz * Software is furnished to do so, subject to the following conditions:
11ed6e6a34SBen Goz *
12ed6e6a34SBen Goz * The above copyright notice and this permission notice shall be included in
13ed6e6a34SBen Goz * all copies or substantial portions of the Software.
14ed6e6a34SBen Goz *
15ed6e6a34SBen Goz * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16ed6e6a34SBen Goz * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17ed6e6a34SBen Goz * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18ed6e6a34SBen Goz * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
19ed6e6a34SBen Goz * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
20ed6e6a34SBen Goz * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
21ed6e6a34SBen Goz * OTHER DEALINGS IN THE SOFTWARE.
22ed6e6a34SBen Goz *
23ed6e6a34SBen Goz */
24ed6e6a34SBen Goz
25ed6e6a34SBen Goz #ifndef KFD_DEVICE_QUEUE_MANAGER_H_
26ed6e6a34SBen Goz #define KFD_DEVICE_QUEUE_MANAGER_H_
27ed6e6a34SBen Goz
28ed6e6a34SBen Goz #include <linux/rwsem.h>
29ed6e6a34SBen Goz #include <linux/list.h>
30efeaed4dSFelix Kuehling #include <linux/mutex.h>
31efeaed4dSFelix Kuehling #include <linux/sched/mm.h>
32ed6e6a34SBen Goz #include "kfd_priv.h"
33ed6e6a34SBen Goz #include "kfd_mqd_manager.h"
34ed6e6a34SBen Goz
35ed6e6a34SBen Goz
36d9d4623cSYong Zhao #define VMID_NUM 16
37d9d4623cSYong Zhao
38cc009e61SMukul Joshi #define KFD_MES_PROCESS_QUANTUM 100000
39cc009e61SMukul Joshi #define KFD_MES_GANG_QUANTUM 10000
407cee6a68SJonathan Kim #define USE_DEFAULT_GRACE_PERIOD 0xffffffff
41cc009e61SMukul Joshi
42ed6e6a34SBen Goz struct device_process_node {
43ed6e6a34SBen Goz struct qcm_process_device *qpd;
44ed6e6a34SBen Goz struct list_head list;
45ed6e6a34SBen Goz };
46ed6e6a34SBen Goz
475bdd3eb2SMukul Joshi union SQ_CMD_BITS {
485bdd3eb2SMukul Joshi struct {
495bdd3eb2SMukul Joshi uint32_t cmd:3;
505bdd3eb2SMukul Joshi uint32_t:1;
515bdd3eb2SMukul Joshi uint32_t mode:3;
525bdd3eb2SMukul Joshi uint32_t check_vmid:1;
535bdd3eb2SMukul Joshi uint32_t trap_id:3;
545bdd3eb2SMukul Joshi uint32_t:5;
555bdd3eb2SMukul Joshi uint32_t wave_id:4;
565bdd3eb2SMukul Joshi uint32_t simd_id:2;
575bdd3eb2SMukul Joshi uint32_t:2;
585bdd3eb2SMukul Joshi uint32_t queue_id:3;
595bdd3eb2SMukul Joshi uint32_t:1;
605bdd3eb2SMukul Joshi uint32_t vm_id:4;
615bdd3eb2SMukul Joshi } bitfields, bits;
625bdd3eb2SMukul Joshi uint32_t u32All;
635bdd3eb2SMukul Joshi signed int i32All;
645bdd3eb2SMukul Joshi float f32All;
655bdd3eb2SMukul Joshi };
665bdd3eb2SMukul Joshi
675bdd3eb2SMukul Joshi union GRBM_GFX_INDEX_BITS {
685bdd3eb2SMukul Joshi struct {
695bdd3eb2SMukul Joshi uint32_t instance_index:8;
705bdd3eb2SMukul Joshi uint32_t sh_index:8;
715bdd3eb2SMukul Joshi uint32_t se_index:8;
725bdd3eb2SMukul Joshi uint32_t:5;
735bdd3eb2SMukul Joshi uint32_t sh_broadcast_writes:1;
745bdd3eb2SMukul Joshi uint32_t instance_broadcast_writes:1;
755bdd3eb2SMukul Joshi uint32_t se_broadcast_writes:1;
765bdd3eb2SMukul Joshi } bitfields, bits;
775bdd3eb2SMukul Joshi uint32_t u32All;
785bdd3eb2SMukul Joshi signed int i32All;
795bdd3eb2SMukul Joshi float f32All;
805bdd3eb2SMukul Joshi };
815bdd3eb2SMukul Joshi
8264c7f8cfSBen Goz /**
8345c9a5e4SOded Gabbay * struct device_queue_manager_ops
8464c7f8cfSBen Goz *
8564c7f8cfSBen Goz * @create_queue: Queue creation routine.
8664c7f8cfSBen Goz *
8764c7f8cfSBen Goz * @destroy_queue: Queue destruction routine.
8864c7f8cfSBen Goz *
8964c7f8cfSBen Goz * @update_queue: Queue update routine.
9064c7f8cfSBen Goz *
9164c7f8cfSBen Goz * @exeute_queues: Dispatches the queues list to the H/W.
9264c7f8cfSBen Goz *
9364c7f8cfSBen Goz * @register_process: This routine associates a specific process with device.
9464c7f8cfSBen Goz *
9564c7f8cfSBen Goz * @unregister_process: destroys the associations between process to device.
9664c7f8cfSBen Goz *
9764c7f8cfSBen Goz * @initialize: Initializes the pipelines and memory module for that device.
9864c7f8cfSBen Goz *
992243f493SRajneesh Bhardwaj * @start: Initializes the resources/modules the device needs for queues
10064c7f8cfSBen Goz * execution. This function is called on device initialization and after the
10164c7f8cfSBen Goz * system woke up after suspension.
10264c7f8cfSBen Goz *
10364c7f8cfSBen Goz * @stop: This routine stops execution of all the active queue running on the
10464c7f8cfSBen Goz * H/W and basically this function called on system suspend.
10564c7f8cfSBen Goz *
10664c7f8cfSBen Goz * @uninitialize: Destroys all the device queue manager resources allocated in
10764c7f8cfSBen Goz * initialize routine.
10864c7f8cfSBen Goz *
10964c7f8cfSBen Goz * @create_kernel_queue: Creates kernel queue. Used for debug queue.
11064c7f8cfSBen Goz *
11164c7f8cfSBen Goz * @destroy_kernel_queue: Destroys kernel queue. Used for debug queue.
11264c7f8cfSBen Goz *
11364c7f8cfSBen Goz * @set_cache_memory_policy: Sets memory policy (cached/ non cached) for the
11464c7f8cfSBen Goz * memory apertures.
11564c7f8cfSBen Goz *
1169fd3f1bfSFelix Kuehling * @process_termination: Clears all process queues belongs to that device.
1179fd3f1bfSFelix Kuehling *
11826103436SFelix Kuehling * @evict_process_queues: Evict all active queues of a process
11926103436SFelix Kuehling *
1202243f493SRajneesh Bhardwaj * @restore_process_queues: Restore all evicted queues of a process
12126103436SFelix Kuehling *
1225df099e8SJay Cornwall * @get_wave_state: Retrieves context save state and optionally copies the
1235df099e8SJay Cornwall * control stack, if kept in the MQD, to the given userspace address.
124dec63443STao Zhou *
125dec63443STao Zhou * @reset_queues: reset queues which consume RAS poison
12642c6c482SDavid Yat Sin * @get_queue_checkpoint_info: Retrieves queue size information for CRIU checkpoint.
12742c6c482SDavid Yat Sin *
12842c6c482SDavid Yat Sin * @checkpoint_mqd: checkpoint queue MQD contents for CRIU.
12964c7f8cfSBen Goz */
13064c7f8cfSBen Goz
13145c9a5e4SOded Gabbay struct device_queue_manager_ops {
132ed6e6a34SBen Goz int (*create_queue)(struct device_queue_manager *dqm,
133ed6e6a34SBen Goz struct queue *q,
1342485c12cSDavid Yat Sin struct qcm_process_device *qpd,
13542c6c482SDavid Yat Sin const struct kfd_criu_queue_priv_data *qd,
1363a9822d7SDavid Yat Sin const void *restore_mqd,
1373a9822d7SDavid Yat Sin const void *restore_ctl_stack);
138992839adSYair Shachar
139ed6e6a34SBen Goz int (*destroy_queue)(struct device_queue_manager *dqm,
140ed6e6a34SBen Goz struct qcm_process_device *qpd,
141ed6e6a34SBen Goz struct queue *q);
142992839adSYair Shachar
143ed6e6a34SBen Goz int (*update_queue)(struct device_queue_manager *dqm,
144c6e559ebSLang Yu struct queue *q, struct mqd_update_info *minfo);
14564c7f8cfSBen Goz
146ed6e6a34SBen Goz int (*register_process)(struct device_queue_manager *dqm,
147ed6e6a34SBen Goz struct qcm_process_device *qpd);
148992839adSYair Shachar
149ed6e6a34SBen Goz int (*unregister_process)(struct device_queue_manager *dqm,
150ed6e6a34SBen Goz struct qcm_process_device *qpd);
151992839adSYair Shachar
152ed6e6a34SBen Goz int (*initialize)(struct device_queue_manager *dqm);
153ed6e6a34SBen Goz int (*start)(struct device_queue_manager *dqm);
154ed6e6a34SBen Goz int (*stop)(struct device_queue_manager *dqm);
15509c34e8dSFelix Kuehling void (*pre_reset)(struct device_queue_manager *dqm);
156ed6e6a34SBen Goz void (*uninitialize)(struct device_queue_manager *dqm);
157ed6e6a34SBen Goz int (*create_kernel_queue)(struct device_queue_manager *dqm,
158ed6e6a34SBen Goz struct kernel_queue *kq,
159ed6e6a34SBen Goz struct qcm_process_device *qpd);
160992839adSYair Shachar
161ed6e6a34SBen Goz void (*destroy_kernel_queue)(struct device_queue_manager *dqm,
162ed6e6a34SBen Goz struct kernel_queue *kq,
163ed6e6a34SBen Goz struct qcm_process_device *qpd);
164992839adSYair Shachar
165ed6e6a34SBen Goz bool (*set_cache_memory_policy)(struct device_queue_manager *dqm,
166ed6e6a34SBen Goz struct qcm_process_device *qpd,
167ed6e6a34SBen Goz enum cache_policy default_policy,
168ed6e6a34SBen Goz enum cache_policy alternate_policy,
169ed6e6a34SBen Goz void __user *alternate_aperture_base,
170ed6e6a34SBen Goz uint64_t alternate_aperture_size);
1719fd3f1bfSFelix Kuehling
1729fd3f1bfSFelix Kuehling int (*process_termination)(struct device_queue_manager *dqm,
1739fd3f1bfSFelix Kuehling struct qcm_process_device *qpd);
17426103436SFelix Kuehling
17526103436SFelix Kuehling int (*evict_process_queues)(struct device_queue_manager *dqm,
17626103436SFelix Kuehling struct qcm_process_device *qpd);
17726103436SFelix Kuehling int (*restore_process_queues)(struct device_queue_manager *dqm,
17826103436SFelix Kuehling struct qcm_process_device *qpd);
1795df099e8SJay Cornwall
1805df099e8SJay Cornwall int (*get_wave_state)(struct device_queue_manager *dqm,
1815df099e8SJay Cornwall struct queue *q,
1825df099e8SJay Cornwall void __user *ctl_stack,
1835df099e8SJay Cornwall u32 *ctl_stack_used_size,
1845df099e8SJay Cornwall u32 *save_area_used_size);
185dec63443STao Zhou
186dec63443STao Zhou int (*reset_queues)(struct device_queue_manager *dqm,
187dec63443STao Zhou uint16_t pasid);
18842c6c482SDavid Yat Sin void (*get_queue_checkpoint_info)(struct device_queue_manager *dqm,
1893a9822d7SDavid Yat Sin const struct queue *q, u32 *mqd_size,
1903a9822d7SDavid Yat Sin u32 *ctl_stack_size);
19142c6c482SDavid Yat Sin
19242c6c482SDavid Yat Sin int (*checkpoint_mqd)(struct device_queue_manager *dqm,
19342c6c482SDavid Yat Sin const struct queue *q,
1943a9822d7SDavid Yat Sin void *mqd,
1953a9822d7SDavid Yat Sin void *ctl_stack);
19645c9a5e4SOded Gabbay };
197ed6e6a34SBen Goz
198d42af779SOded Gabbay struct device_queue_manager_asic_ops {
199bfd5e378SYong Zhao int (*update_qpd)(struct device_queue_manager *dqm,
200d42af779SOded Gabbay struct qcm_process_device *qpd);
201d42af779SOded Gabbay bool (*set_cache_memory_policy)(struct device_queue_manager *dqm,
202d42af779SOded Gabbay struct qcm_process_device *qpd,
203d42af779SOded Gabbay enum cache_policy default_policy,
204d42af779SOded Gabbay enum cache_policy alternate_policy,
205d42af779SOded Gabbay void __user *alternate_aperture_base,
206d42af779SOded Gabbay uint64_t alternate_aperture_size);
2073e3f6e1aSOded Gabbay void (*init_sdma_vm)(struct device_queue_manager *dqm,
2083e3f6e1aSOded Gabbay struct queue *q,
2093e3f6e1aSOded Gabbay struct qcm_process_device *qpd);
210972fcdb5SOak Zeng struct mqd_manager * (*mqd_manager_init)(enum KFD_MQD_TYPE type,
2118dc1db31SMukul Joshi struct kfd_node *dev);
212d42af779SOded Gabbay };
213d42af779SOded Gabbay
21445c9a5e4SOded Gabbay /**
21545c9a5e4SOded Gabbay * struct device_queue_manager
21645c9a5e4SOded Gabbay *
21745c9a5e4SOded Gabbay * This struct is a base class for the kfd queues scheduler in the
21845c9a5e4SOded Gabbay * device level. The device base class should expose the basic operations
21945c9a5e4SOded Gabbay * for queue creation and queue destruction. This base class hides the
22045c9a5e4SOded Gabbay * scheduling mode of the driver and the specific implementation of the
22145c9a5e4SOded Gabbay * concrete device. This class is the only class in the queues scheduler
22245c9a5e4SOded Gabbay * that configures the H/W.
22345c9a5e4SOded Gabbay *
22445c9a5e4SOded Gabbay */
22545c9a5e4SOded Gabbay
22645c9a5e4SOded Gabbay struct device_queue_manager {
22745c9a5e4SOded Gabbay struct device_queue_manager_ops ops;
228bfd5e378SYong Zhao struct device_queue_manager_asic_ops asic_ops;
229ed6e6a34SBen Goz
2308d5f3552SYong Zhao struct mqd_manager *mqd_mgrs[KFD_MQD_TYPE_MAX];
2319af5379cSOak Zeng struct packet_manager packet_mgr;
2328dc1db31SMukul Joshi struct kfd_node *dev;
233efeaed4dSFelix Kuehling struct mutex lock_hidden; /* use dqm_lock/unlock(dqm) */
234ed6e6a34SBen Goz struct list_head queues;
235efeaed4dSFelix Kuehling unsigned int saved_flags;
236ed6e6a34SBen Goz unsigned int processes_count;
23781b820b3SYong Zhao unsigned int active_queue_count;
238b42902f4SYong Zhao unsigned int active_cp_queue_count;
239b8020b03SJoseph Greathouse unsigned int gws_queue_count;
240b8cbab04SOded Gabbay unsigned int total_queue_count;
241ed6e6a34SBen Goz unsigned int next_pipe_to_allocate;
242ed6e6a34SBen Goz unsigned int *allocated_queues;
243a805889aSMukul Joshi DECLARE_BITMAP(sdma_bitmap, KFD_MAX_SDMA_QUEUES);
244a805889aSMukul Joshi DECLARE_BITMAP(xgmi_sdma_bitmap, KFD_MAX_SDMA_QUEUES);
245d9d4623cSYong Zhao /* the pasid mapping for each kfd vmid */
246d9d4623cSYong Zhao uint16_t vmid_pasid[VMID_NUM];
247ed6e6a34SBen Goz uint64_t pipelines_addr;
248ed6e6a34SBen Goz uint64_t fence_gpu_addr;
249b010affeSQu Huang uint64_t *fence_addr;
250ed6e6a34SBen Goz struct kfd_mem_obj *fence_mem;
251ed6e6a34SBen Goz bool active_runlist;
252d146c5a7SFelix Kuehling int sched_policy;
25397ae3c8cSJonathan Kim uint32_t trap_debug_vmid;
25473ea648dSShaoyun Liu
25573ea648dSShaoyun Liu /* hw exception */
25673ea648dSShaoyun Liu bool is_hws_hang;
25709c34e8dSFelix Kuehling bool is_resetting;
25873ea648dSShaoyun Liu struct work_struct hw_exception_work;
25911614c36SOak Zeng struct kfd_mem_obj hiq_sdma_mqd;
2602c99a547SPhilip Yang bool sched_running;
261c2d43918SMukul Joshi
262c2d43918SMukul Joshi /* used for GFX 9.4.3 only */
263c2d43918SMukul Joshi uint32_t current_logical_xcc_start;
2647cee6a68SJonathan Kim
2657cee6a68SJonathan Kim uint32_t wait_times;
266a70a93faSJonathan Kim
267a70a93faSJonathan Kim wait_queue_head_t destroy_wait;
268ed6e6a34SBen Goz };
269ed6e6a34SBen Goz
270bfd5e378SYong Zhao void device_queue_manager_init_cik(
271bfd5e378SYong Zhao struct device_queue_manager_asic_ops *asic_ops);
272bfd5e378SYong Zhao void device_queue_manager_init_vi(
273bfd5e378SYong Zhao struct device_queue_manager_asic_ops *asic_ops);
274bed4f110SFelix Kuehling void device_queue_manager_init_v9(
275bed4f110SFelix Kuehling struct device_queue_manager_asic_ops *asic_ops);
276*80e28aafSAlex Deucher void device_queue_manager_init_v10(
27714328aa5SPhilip Cox struct device_queue_manager_asic_ops *asic_ops);
278cc009e61SMukul Joshi void device_queue_manager_init_v11(
279cc009e61SMukul Joshi struct device_queue_manager_asic_ops *asic_ops);
280a22fc854SBen Goz void program_sh_mem_settings(struct device_queue_manager *dqm,
281a22fc854SBen Goz struct qcm_process_device *qpd);
282e6945304SYong Zhao unsigned int get_cp_queues_num(struct device_queue_manager *dqm);
283d0b63bb3SAndres Rodriguez unsigned int get_queues_per_pipe(struct device_queue_manager *dqm);
284d0b63bb3SAndres Rodriguez unsigned int get_pipes_per_mec(struct device_queue_manager *dqm);
28598bb9222SYong Zhao unsigned int get_num_sdma_queues(struct device_queue_manager *dqm);
2861b4670f6SOak Zeng unsigned int get_num_xgmi_sdma_queues(struct device_queue_manager *dqm);
28797ae3c8cSJonathan Kim int reserve_debug_trap_vmid(struct device_queue_manager *dqm,
28897ae3c8cSJonathan Kim struct qcm_process_device *qpd);
28997ae3c8cSJonathan Kim int release_debug_trap_vmid(struct device_queue_manager *dqm,
29097ae3c8cSJonathan Kim struct qcm_process_device *qpd);
291a70a93faSJonathan Kim int suspend_queues(struct kfd_process *p,
292a70a93faSJonathan Kim uint32_t num_queues,
293a70a93faSJonathan Kim uint32_t grace_period,
294a70a93faSJonathan Kim uint64_t exception_clear_mask,
295a70a93faSJonathan Kim uint32_t *usr_queue_id_array);
296a70a93faSJonathan Kim int resume_queues(struct kfd_process *p,
297a70a93faSJonathan Kim uint32_t num_queues,
298a70a93faSJonathan Kim uint32_t *usr_queue_id_array);
299b17bd5dbSJonathan Kim void set_queue_snapshot_entry(struct queue *q,
300b17bd5dbSJonathan Kim uint64_t exception_clear_mask,
301b17bd5dbSJonathan Kim struct kfd_queue_snapshot_entry *qss_entry);
3020de4ec9aSJonathan Kim int debug_lock_and_unmap(struct device_queue_manager *dqm);
3030de4ec9aSJonathan Kim int debug_map_and_unlock(struct device_queue_manager *dqm);
3040de4ec9aSJonathan Kim int debug_refresh_runlist(struct device_queue_manager *dqm);
3050b3674aeSOded Gabbay
get_sh_mem_bases_32(struct kfd_process_device * pdd)306a104299bSDaniel Vetter static inline unsigned int get_sh_mem_bases_32(struct kfd_process_device *pdd)
3070b3674aeSOded Gabbay {
3080b3674aeSOded Gabbay return (pdd->lds_base >> 16) & 0xFF;
3090b3674aeSOded Gabbay }
3100b3674aeSOded Gabbay
311a104299bSDaniel Vetter static inline unsigned int
get_sh_mem_bases_nybble_64(struct kfd_process_device * pdd)3120b3674aeSOded Gabbay get_sh_mem_bases_nybble_64(struct kfd_process_device *pdd)
3130b3674aeSOded Gabbay {
3140b3674aeSOded Gabbay return (pdd->lds_base >> 60) & 0x0E;
3150b3674aeSOded Gabbay }
3160b3674aeSOded Gabbay
317efeaed4dSFelix Kuehling /* The DQM lock can be taken in MMU notifiers. Make sure no reclaim-FS
318efeaed4dSFelix Kuehling * happens while holding this lock anywhere to prevent deadlocks when
319efeaed4dSFelix Kuehling * an MMU notifier runs in reclaim-FS context.
320efeaed4dSFelix Kuehling */
dqm_lock(struct device_queue_manager * dqm)321efeaed4dSFelix Kuehling static inline void dqm_lock(struct device_queue_manager *dqm)
322efeaed4dSFelix Kuehling {
323efeaed4dSFelix Kuehling mutex_lock(&dqm->lock_hidden);
3241fb8b1fcSFelix Kuehling dqm->saved_flags = memalloc_noreclaim_save();
325efeaed4dSFelix Kuehling }
dqm_unlock(struct device_queue_manager * dqm)326efeaed4dSFelix Kuehling static inline void dqm_unlock(struct device_queue_manager *dqm)
327efeaed4dSFelix Kuehling {
3281fb8b1fcSFelix Kuehling memalloc_noreclaim_restore(dqm->saved_flags);
329efeaed4dSFelix Kuehling mutex_unlock(&dqm->lock_hidden);
330efeaed4dSFelix Kuehling }
331efeaed4dSFelix Kuehling
read_sdma_queue_counter(uint64_t __user * q_rptr,uint64_t * val)332818b0324SMukul Joshi static inline int read_sdma_queue_counter(uint64_t __user *q_rptr, uint64_t *val)
333818b0324SMukul Joshi {
3342243f493SRajneesh Bhardwaj /* SDMA activity counter is stored at queue's RPTR + 0x8 location. */
335818b0324SMukul Joshi return get_user(*val, q_rptr + 1);
336818b0324SMukul Joshi }
337ed6e6a34SBen Goz #endif /* KFD_DEVICE_QUEUE_MANAGER_H_ */
338