1 /*
2  * Copyright 2014 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  */
23 
24 #ifndef KFD_DEVICE_QUEUE_MANAGER_H_
25 #define KFD_DEVICE_QUEUE_MANAGER_H_
26 
27 #include <linux/rwsem.h>
28 #include <linux/list.h>
29 #include "kfd_priv.h"
30 #include "kfd_mqd_manager.h"
31 
32 #define QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS	(500)
33 #define QUEUES_PER_PIPE				(8)
34 #define PIPE_PER_ME_CP_SCHEDULING		(3)
35 #define CIK_VMID_NUM				(8)
36 #define KFD_VMID_START_OFFSET			(8)
37 #define VMID_PER_DEVICE				CIK_VMID_NUM
38 #define KFD_DQM_FIRST_PIPE			(0)
39 #define CIK_SDMA_QUEUES				(4)
40 #define CIK_SDMA_QUEUES_PER_ENGINE		(2)
41 #define CIK_SDMA_ENGINE_NUM			(2)
42 
43 struct device_process_node {
44 	struct qcm_process_device *qpd;
45 	struct list_head list;
46 };
47 
48 /**
49  * struct device_queue_manager_ops
50  *
51  * @create_queue: Queue creation routine.
52  *
53  * @destroy_queue: Queue destruction routine.
54  *
55  * @update_queue: Queue update routine.
56  *
57  * @get_mqd_manager: Returns the mqd manager according to the mqd type.
58  *
59  * @exeute_queues: Dispatches the queues list to the H/W.
60  *
61  * @register_process: This routine associates a specific process with device.
62  *
63  * @unregister_process: destroys the associations between process to device.
64  *
65  * @initialize: Initializes the pipelines and memory module for that device.
66  *
67  * @start: Initializes the resources/modules the the device needs for queues
68  * execution. This function is called on device initialization and after the
69  * system woke up after suspension.
70  *
71  * @stop: This routine stops execution of all the active queue running on the
72  * H/W and basically this function called on system suspend.
73  *
74  * @uninitialize: Destroys all the device queue manager resources allocated in
75  * initialize routine.
76  *
77  * @create_kernel_queue: Creates kernel queue. Used for debug queue.
78  *
79  * @destroy_kernel_queue: Destroys kernel queue. Used for debug queue.
80  *
81  * @set_cache_memory_policy: Sets memory policy (cached/ non cached) for the
82  * memory apertures.
83  *
84  */
85 
86 struct device_queue_manager_ops {
87 	int	(*create_queue)(struct device_queue_manager *dqm,
88 				struct queue *q,
89 				struct qcm_process_device *qpd,
90 				int *allocate_vmid);
91 
92 	int	(*destroy_queue)(struct device_queue_manager *dqm,
93 				struct qcm_process_device *qpd,
94 				struct queue *q);
95 
96 	int	(*update_queue)(struct device_queue_manager *dqm,
97 				struct queue *q);
98 
99 	struct mqd_manager * (*get_mqd_manager)
100 					(struct device_queue_manager *dqm,
101 					enum KFD_MQD_TYPE type);
102 
103 	int	(*register_process)(struct device_queue_manager *dqm,
104 					struct qcm_process_device *qpd);
105 
106 	int	(*unregister_process)(struct device_queue_manager *dqm,
107 					struct qcm_process_device *qpd);
108 
109 	int	(*initialize)(struct device_queue_manager *dqm);
110 	int	(*start)(struct device_queue_manager *dqm);
111 	int	(*stop)(struct device_queue_manager *dqm);
112 	void	(*uninitialize)(struct device_queue_manager *dqm);
113 	int	(*create_kernel_queue)(struct device_queue_manager *dqm,
114 					struct kernel_queue *kq,
115 					struct qcm_process_device *qpd);
116 
117 	void	(*destroy_kernel_queue)(struct device_queue_manager *dqm,
118 					struct kernel_queue *kq,
119 					struct qcm_process_device *qpd);
120 
121 	bool	(*set_cache_memory_policy)(struct device_queue_manager *dqm,
122 					   struct qcm_process_device *qpd,
123 					   enum cache_policy default_policy,
124 					   enum cache_policy alternate_policy,
125 					   void __user *alternate_aperture_base,
126 					   uint64_t alternate_aperture_size);
127 };
128 
129 struct device_queue_manager_asic_ops {
130 	int	(*register_process)(struct device_queue_manager *dqm,
131 					struct qcm_process_device *qpd);
132 	int	(*initialize)(struct device_queue_manager *dqm);
133 	bool	(*set_cache_memory_policy)(struct device_queue_manager *dqm,
134 					   struct qcm_process_device *qpd,
135 					   enum cache_policy default_policy,
136 					   enum cache_policy alternate_policy,
137 					   void __user *alternate_aperture_base,
138 					   uint64_t alternate_aperture_size);
139 	void	(*init_sdma_vm)(struct device_queue_manager *dqm,
140 				struct queue *q,
141 				struct qcm_process_device *qpd);
142 };
143 
144 /**
145  * struct device_queue_manager
146  *
147  * This struct is a base class for the kfd queues scheduler in the
148  * device level. The device base class should expose the basic operations
149  * for queue creation and queue destruction. This base class hides the
150  * scheduling mode of the driver and the specific implementation of the
151  * concrete device. This class is the only class in the queues scheduler
152  * that configures the H/W.
153  *
154  */
155 
156 struct device_queue_manager {
157 	struct device_queue_manager_ops ops;
158 	struct device_queue_manager_asic_ops ops_asic_specific;
159 
160 	struct mqd_manager	*mqds[KFD_MQD_TYPE_MAX];
161 	struct packet_manager	packets;
162 	struct kfd_dev		*dev;
163 	struct mutex		lock;
164 	struct list_head	queues;
165 	unsigned int		processes_count;
166 	unsigned int		queue_count;
167 	unsigned int		sdma_queue_count;
168 	unsigned int		total_queue_count;
169 	unsigned int		next_pipe_to_allocate;
170 	unsigned int		*allocated_queues;
171 	unsigned int		sdma_bitmap;
172 	unsigned int		vmid_bitmap;
173 	uint64_t		pipelines_addr;
174 	struct kfd_mem_obj	*pipeline_mem;
175 	uint64_t		fence_gpu_addr;
176 	unsigned int		*fence_addr;
177 	struct kfd_mem_obj	*fence_mem;
178 	bool			active_runlist;
179 };
180 
181 void device_queue_manager_init_cik(struct device_queue_manager_asic_ops *ops);
182 void device_queue_manager_init_vi(struct device_queue_manager_asic_ops *ops);
183 void program_sh_mem_settings(struct device_queue_manager *dqm,
184 					struct qcm_process_device *qpd);
185 int init_pipelines(struct device_queue_manager *dqm,
186 		unsigned int pipes_num, unsigned int first_pipe);
187 unsigned int get_first_pipe(struct device_queue_manager *dqm);
188 unsigned int get_pipes_num(struct device_queue_manager *dqm);
189 
190 extern inline unsigned int get_sh_mem_bases_32(struct kfd_process_device *pdd)
191 {
192 	return (pdd->lds_base >> 16) & 0xFF;
193 }
194 
195 extern inline unsigned int
196 get_sh_mem_bases_nybble_64(struct kfd_process_device *pdd)
197 {
198 	return (pdd->lds_base >> 60) & 0x0E;
199 }
200 
201 #endif /* KFD_DEVICE_QUEUE_MANAGER_H_ */
202