1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright 2022 Advanced Micro Devices, Inc.
4  *
5  * Permission is hereby granted, free of charge, to any person obtaining a
6  * copy of this software and associated documentation files (the "Software"),
7  * to deal in the Software without restriction, including without limitation
8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9  * and/or sell copies of the Software, and to permit persons to whom the
10  * Software is furnished to do so, subject to the following conditions:
11  *
12  * The above copyright notice and this permission notice shall be included in
13  * all copies or substantial portions of the Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
19  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
20  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
21  * OTHER DEALINGS IN THE SOFTWARE.
22  *
23  */
24 
25 #include "amdgpu.h"
26 
27 /**
28  * amdgpu_mm_rdoorbell - read a doorbell dword
29  *
30  * @adev: amdgpu_device pointer
31  * @index: doorbell index
32  *
33  * Returns the value in the doorbell aperture at the
34  * requested doorbell index (CIK).
35  */
36 u32 amdgpu_mm_rdoorbell(struct amdgpu_device *adev, u32 index)
37 {
38 	if (amdgpu_device_skip_hw_access(adev))
39 		return 0;
40 
41 	if (index < adev->doorbell.num_kernel_doorbells)
42 		return readl(adev->doorbell.cpu_addr + index);
43 
44 	DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
45 	return 0;
46 }
47 
48 /**
49  * amdgpu_mm_wdoorbell - write a doorbell dword
50  *
51  * @adev: amdgpu_device pointer
52  * @index: doorbell index
53  * @v: value to write
54  *
55  * Writes @v to the doorbell aperture at the
56  * requested doorbell index (CIK).
57  */
58 void amdgpu_mm_wdoorbell(struct amdgpu_device *adev, u32 index, u32 v)
59 {
60 	if (amdgpu_device_skip_hw_access(adev))
61 		return;
62 
63 	if (index < adev->doorbell.num_kernel_doorbells)
64 		writel(v, adev->doorbell.cpu_addr + index);
65 	else
66 		DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
67 }
68 
69 /**
70  * amdgpu_mm_rdoorbell64 - read a doorbell Qword
71  *
72  * @adev: amdgpu_device pointer
73  * @index: doorbell index
74  *
75  * Returns the value in the doorbell aperture at the
76  * requested doorbell index (VEGA10+).
77  */
78 u64 amdgpu_mm_rdoorbell64(struct amdgpu_device *adev, u32 index)
79 {
80 	if (amdgpu_device_skip_hw_access(adev))
81 		return 0;
82 
83 	if (index < adev->doorbell.num_kernel_doorbells)
84 		return atomic64_read((atomic64_t *)(adev->doorbell.cpu_addr + index));
85 
86 	DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
87 	return 0;
88 }
89 
90 /**
91  * amdgpu_mm_wdoorbell64 - write a doorbell Qword
92  *
93  * @adev: amdgpu_device pointer
94  * @index: doorbell index
95  * @v: value to write
96  *
97  * Writes @v to the doorbell aperture at the
98  * requested doorbell index (VEGA10+).
99  */
100 void amdgpu_mm_wdoorbell64(struct amdgpu_device *adev, u32 index, u64 v)
101 {
102 	if (amdgpu_device_skip_hw_access(adev))
103 		return;
104 
105 	if (index < adev->doorbell.num_kernel_doorbells)
106 		atomic64_set((atomic64_t *)(adev->doorbell.cpu_addr + index), v);
107 	else
108 		DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
109 }
110 
111 /**
112  * amdgpu_doorbell_index_on_bar - Find doorbell's absolute offset in BAR
113  *
114  * @adev: amdgpu_device pointer
115  * @db_bo: doorbell object's bo
116  * @db_index: doorbell relative index in this doorbell object
117  *
118  * returns doorbell's absolute index in BAR
119  */
120 uint32_t amdgpu_doorbell_index_on_bar(struct amdgpu_device *adev,
121 				       struct amdgpu_bo *db_bo,
122 				       uint32_t doorbell_index)
123 {
124 	int db_bo_offset;
125 
126 	db_bo_offset = amdgpu_bo_gpu_offset_no_check(db_bo);
127 
128 	/* doorbell index is 32 bit but doorbell's size is 64-bit, so *2 */
129 	return db_bo_offset / sizeof(u32) + doorbell_index * 2;
130 }
131 
132 /**
133  * amdgpu_doorbell_create_kernel_doorbells - Create kernel doorbells for graphics
134  *
135  * @adev: amdgpu_device pointer
136  *
137  * Creates doorbells for graphics driver usages.
138  * returns 0 on success, error otherwise.
139  */
140 int amdgpu_doorbell_create_kernel_doorbells(struct amdgpu_device *adev)
141 {
142 	int r;
143 	int size;
144 
145 	/* SI HW does not have doorbells, skip allocation */
146 	if (adev->doorbell.num_kernel_doorbells == 0)
147 		return 0;
148 
149 	/* Reserve first num_kernel_doorbells (page-aligned) for kernel ops */
150 	size = ALIGN(adev->doorbell.num_kernel_doorbells * sizeof(u32), PAGE_SIZE);
151 
152 	/* Allocate an extra page for MES kernel usages (ring test) */
153 	adev->mes.db_start_dw_offset = size / sizeof(u32);
154 	size += PAGE_SIZE;
155 
156 	r = amdgpu_bo_create_kernel(adev,
157 				    size,
158 				    PAGE_SIZE,
159 				    AMDGPU_GEM_DOMAIN_DOORBELL,
160 				    &adev->doorbell.kernel_doorbells,
161 				    NULL,
162 				    (void **)&adev->doorbell.cpu_addr);
163 	if (r) {
164 		DRM_ERROR("Failed to allocate kernel doorbells, err=%d\n", r);
165 		return r;
166 	}
167 
168 	adev->doorbell.num_kernel_doorbells = size / sizeof(u32);
169 	return 0;
170 }
171 
172 /*
173  * GPU doorbell aperture helpers function.
174  */
175 /**
176  * amdgpu_doorbell_init - Init doorbell driver information.
177  *
178  * @adev: amdgpu_device pointer
179  *
180  * Init doorbell driver information (CIK)
181  * Returns 0 on success, error on failure.
182  */
183 int amdgpu_doorbell_init(struct amdgpu_device *adev)
184 {
185 
186 	/* No doorbell on SI hardware generation */
187 	if (adev->asic_type < CHIP_BONAIRE) {
188 		adev->doorbell.base = 0;
189 		adev->doorbell.size = 0;
190 		adev->doorbell.num_kernel_doorbells = 0;
191 		return 0;
192 	}
193 
194 	if (pci_resource_flags(adev->pdev, 2) & IORESOURCE_UNSET)
195 		return -EINVAL;
196 
197 	amdgpu_asic_init_doorbell_index(adev);
198 
199 	/* doorbell bar mapping */
200 	adev->doorbell.base = pci_resource_start(adev->pdev, 2);
201 	adev->doorbell.size = pci_resource_len(adev->pdev, 2);
202 
203 	adev->doorbell.num_kernel_doorbells =
204 		min_t(u32, adev->doorbell.size / sizeof(u32),
205 		      adev->doorbell_index.max_assignment + 1);
206 	if (adev->doorbell.num_kernel_doorbells == 0)
207 		return -EINVAL;
208 
209 	/*
210 	 * For Vega, reserve and map two pages on doorbell BAR since SDMA
211 	 * paging queue doorbell use the second page. The
212 	 * AMDGPU_DOORBELL64_MAX_ASSIGNMENT definition assumes all the
213 	 * doorbells are in the first page. So with paging queue enabled,
214 	 * the max num_kernel_doorbells should + 1 page (0x400 in dword)
215 	 */
216 	if (adev->asic_type >= CHIP_VEGA10)
217 		adev->doorbell.num_kernel_doorbells += 0x400;
218 
219 	return 0;
220 }
221 
222 /**
223  * amdgpu_doorbell_fini - Tear down doorbell driver information.
224  *
225  * @adev: amdgpu_device pointer
226  *
227  * Tear down doorbell driver information (CIK)
228  */
229 void amdgpu_doorbell_fini(struct amdgpu_device *adev)
230 {
231 	amdgpu_bo_free_kernel(&adev->doorbell.kernel_doorbells,
232 			      NULL,
233 			      (void **)&adev->doorbell.cpu_addr);
234 }
235