142de677fSPhilip Yang /* SPDX-License-Identifier: GPL-2.0 OR MIT */
242de677fSPhilip Yang /*
342de677fSPhilip Yang * Copyright 2020-2021 Advanced Micro Devices, Inc.
442de677fSPhilip Yang *
542de677fSPhilip Yang * Permission is hereby granted, free of charge, to any person obtaining a
642de677fSPhilip Yang * copy of this software and associated documentation files (the "Software"),
742de677fSPhilip Yang * to deal in the Software without restriction, including without limitation
842de677fSPhilip Yang * the rights to use, copy, modify, merge, publish, distribute, sublicense,
942de677fSPhilip Yang * and/or sell copies of the Software, and to permit persons to whom the
1042de677fSPhilip Yang * Software is furnished to do so, subject to the following conditions:
1142de677fSPhilip Yang *
1242de677fSPhilip Yang * The above copyright notice and this permission notice shall be included in
1342de677fSPhilip Yang * all copies or substantial portions of the Software.
1442de677fSPhilip Yang *
1542de677fSPhilip Yang * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
1642de677fSPhilip Yang * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
1742de677fSPhilip Yang * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
1842de677fSPhilip Yang * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
1942de677fSPhilip Yang * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
2042de677fSPhilip Yang * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
2142de677fSPhilip Yang * OTHER DEALINGS IN THE SOFTWARE.
2242de677fSPhilip Yang *
2342de677fSPhilip Yang */
2442de677fSPhilip Yang
2542de677fSPhilip Yang #ifndef KFD_SVM_H_
2642de677fSPhilip Yang #define KFD_SVM_H_
2742de677fSPhilip Yang
284ab159d2SFelix Kuehling #if IS_ENABLED(CONFIG_HSA_AMD_SVM)
294ab159d2SFelix Kuehling
3042de677fSPhilip Yang #include <linux/rwsem.h>
3142de677fSPhilip Yang #include <linux/list.h>
3242de677fSPhilip Yang #include <linux/mutex.h>
3342de677fSPhilip Yang #include <linux/sched/mm.h>
3442de677fSPhilip Yang #include <linux/hmm.h>
3542de677fSPhilip Yang #include "amdgpu.h"
3642de677fSPhilip Yang #include "kfd_priv.h"
3742de677fSPhilip Yang
381d5dbfe6SAlex Sierra #define SVM_RANGE_VRAM_DOMAIN (1UL << 0)
393a61dae8SAlex Sierra #define SVM_ADEV_PGMAP_OWNER(adev)\
403a61dae8SAlex Sierra ((adev)->hive ? (void *)(adev)->hive : (void *)(adev))
413a61dae8SAlex Sierra
42e49fe404SFelix Kuehling struct svm_range_bo {
43e49fe404SFelix Kuehling struct amdgpu_bo *bo;
44e49fe404SFelix Kuehling struct kref kref;
45e49fe404SFelix Kuehling struct list_head range_list; /* all svm ranges shared this bo */
46e49fe404SFelix Kuehling spinlock_t list_lock;
47b41896e3SFelix Kuehling struct amdgpu_amdkfd_fence *eviction_fence;
48b41896e3SFelix Kuehling struct work_struct eviction_work;
49b41896e3SFelix Kuehling uint32_t evicting;
5069879b30SPhilip Yang struct work_struct release_work;
515fb34bd9SAlex Sierra struct kfd_node *node;
52e49fe404SFelix Kuehling };
53e49fe404SFelix Kuehling
544683cfecSPhilip Yang enum svm_work_list_ops {
554683cfecSPhilip Yang SVM_OP_NULL,
564683cfecSPhilip Yang SVM_OP_UNMAP_RANGE,
574683cfecSPhilip Yang SVM_OP_UPDATE_RANGE_NOTIFIER,
5890d7d3edSFelix Kuehling SVM_OP_UPDATE_RANGE_NOTIFIER_AND_MAP,
5990d7d3edSFelix Kuehling SVM_OP_ADD_RANGE,
6090d7d3edSFelix Kuehling SVM_OP_ADD_RANGE_AND_MAP
614683cfecSPhilip Yang };
624683cfecSPhilip Yang
634683cfecSPhilip Yang struct svm_work_list_item {
644683cfecSPhilip Yang enum svm_work_list_ops op;
654683cfecSPhilip Yang struct mm_struct *mm;
664683cfecSPhilip Yang };
674683cfecSPhilip Yang
6842de677fSPhilip Yang /**
6942de677fSPhilip Yang * struct svm_range - shared virtual memory range
7042de677fSPhilip Yang *
7142de677fSPhilip Yang * @svms: list of svm ranges, structure defined in kfd_process
720b0e518dSFelix Kuehling * @migrate_mutex: to serialize range migration, validation and mapping update
7342de677fSPhilip Yang * @start: range start address in pages
7442de677fSPhilip Yang * @last: range last address in pages
7542de677fSPhilip Yang * @it_node: node [start, last] stored in interval tree, start, last are page
7642de677fSPhilip Yang * aligned, page size is (last - start + 1)
7742de677fSPhilip Yang * @list: link list node, used to scan all ranges of svms
7842de677fSPhilip Yang * @update_list:link list node used to add to update_list
79f80fe9d3SFelix Kuehling * @mapping: bo_va mapping structure to create and update GPU page table
8042de677fSPhilip Yang * @npages: number of pages
81f80fe9d3SFelix Kuehling * @dma_addr: dma mapping address on each GPU for system memory physical page
82e49fe404SFelix Kuehling * @ttm_res: vram ttm resource map
83e49fe404SFelix Kuehling * @offset: range start offset within mm_nodes
84e49fe404SFelix Kuehling * @svm_bo: struct to manage splited amdgpu_bo
85e49fe404SFelix Kuehling * @svm_bo_list:link list node, to scan all ranges which share same svm_bo
86b1c46c7dSPhilip Yang * @lock: protect prange start, last, child_list, svm_bo_list
87b1c46c7dSPhilip Yang * @saved_flags:save/restore current PF_MEMALLOC flags
8842de677fSPhilip Yang * @flags: flags defined as KFD_IOCTL_SVM_FLAG_*
8942de677fSPhilip Yang * @perferred_loc: perferred location, 0 for CPU, or GPU id
9042de677fSPhilip Yang * @perfetch_loc: last prefetch location, 0 for CPU, or GPU id
9142de677fSPhilip Yang * @actual_loc: the actual location, 0 for CPU, or GPU id
9242de677fSPhilip Yang * @granularity:migration granularity, log2 num pages
938a7c184aSFelix Kuehling * @invalid: not 0 means cpu page table is invalidated
94564d2b92SFelix Kuehling * @validate_timestamp: system timestamp when range is validated
95b1c46c7dSPhilip Yang * @notifier: register mmu interval notifier
964683cfecSPhilip Yang * @work_item: deferred work item information
974683cfecSPhilip Yang * @deferred_list: list header used to add range to deferred list
984683cfecSPhilip Yang * @child_list: list header for split ranges which are not added to svms yet
9942de677fSPhilip Yang * @bitmap_access: index bitmap of GPUs which can access the range
10042de677fSPhilip Yang * @bitmap_aip: index bitmap of GPUs which can access the range in place
10142de677fSPhilip Yang *
10242de677fSPhilip Yang * Data structure for virtual memory range shared by CPU and GPUs, it can be
10342de677fSPhilip Yang * allocated from system memory ram or device vram, and migrate from ram to vram
10442de677fSPhilip Yang * or from vram to ram.
10542de677fSPhilip Yang */
10642de677fSPhilip Yang struct svm_range {
10742de677fSPhilip Yang struct svm_range_list *svms;
1080b0e518dSFelix Kuehling struct mutex migrate_mutex;
10942de677fSPhilip Yang unsigned long start;
11042de677fSPhilip Yang unsigned long last;
11142de677fSPhilip Yang struct interval_tree_node it_node;
11242de677fSPhilip Yang struct list_head list;
11342de677fSPhilip Yang struct list_head update_list;
11442de677fSPhilip Yang uint64_t npages;
115f80fe9d3SFelix Kuehling dma_addr_t *dma_addr[MAX_GPU_INSTANCE];
116e49fe404SFelix Kuehling struct ttm_resource *ttm_res;
117e49fe404SFelix Kuehling uint64_t offset;
118e49fe404SFelix Kuehling struct svm_range_bo *svm_bo;
119e49fe404SFelix Kuehling struct list_head svm_bo_list;
120b1c46c7dSPhilip Yang struct mutex lock;
121b1c46c7dSPhilip Yang unsigned int saved_flags;
12242de677fSPhilip Yang uint32_t flags;
12342de677fSPhilip Yang uint32_t preferred_loc;
12442de677fSPhilip Yang uint32_t prefetch_loc;
12542de677fSPhilip Yang uint32_t actual_loc;
12642de677fSPhilip Yang uint8_t granularity;
1278a7c184aSFelix Kuehling atomic_t invalid;
128e0f1e65bSPhilip Yang ktime_t validate_timestamp;
129b1c46c7dSPhilip Yang struct mmu_interval_notifier notifier;
1304683cfecSPhilip Yang struct svm_work_list_item work_item;
1314683cfecSPhilip Yang struct list_head deferred_list;
1324683cfecSPhilip Yang struct list_head child_list;
13342de677fSPhilip Yang DECLARE_BITMAP(bitmap_access, MAX_GPU_INSTANCE);
13442de677fSPhilip Yang DECLARE_BITMAP(bitmap_aip, MAX_GPU_INSTANCE);
1356b9c63a6SPhilip Yang bool mapped_to_gpu;
13642de677fSPhilip Yang };
13742de677fSPhilip Yang
svm_range_lock(struct svm_range * prange)138b1c46c7dSPhilip Yang static inline void svm_range_lock(struct svm_range *prange)
139b1c46c7dSPhilip Yang {
140b1c46c7dSPhilip Yang mutex_lock(&prange->lock);
141b1c46c7dSPhilip Yang prange->saved_flags = memalloc_noreclaim_save();
142b1c46c7dSPhilip Yang
143b1c46c7dSPhilip Yang }
svm_range_unlock(struct svm_range * prange)144b1c46c7dSPhilip Yang static inline void svm_range_unlock(struct svm_range *prange)
145b1c46c7dSPhilip Yang {
146b1c46c7dSPhilip Yang memalloc_noreclaim_restore(prange->saved_flags);
147b1c46c7dSPhilip Yang mutex_unlock(&prange->lock);
148b1c46c7dSPhilip Yang }
149b1c46c7dSPhilip Yang
svm_range_bo_ref(struct svm_range_bo * svm_bo)1507981ec65SAlex Sierra static inline struct svm_range_bo *svm_range_bo_ref(struct svm_range_bo *svm_bo)
1517981ec65SAlex Sierra {
1527981ec65SAlex Sierra if (svm_bo)
1537981ec65SAlex Sierra kref_get(&svm_bo->kref);
1547981ec65SAlex Sierra
1557981ec65SAlex Sierra return svm_bo;
1567981ec65SAlex Sierra }
1577981ec65SAlex Sierra
15842de677fSPhilip Yang int svm_range_list_init(struct kfd_process *p);
15942de677fSPhilip Yang void svm_range_list_fini(struct kfd_process *p);
16042de677fSPhilip Yang int svm_ioctl(struct kfd_process *p, enum kfd_ioctl_svm_op op, uint64_t start,
16142de677fSPhilip Yang uint64_t size, uint32_t nattrs,
16242de677fSPhilip Yang struct kfd_ioctl_svm_attribute *attrs);
16348ff079bSFelix Kuehling struct svm_range *svm_range_from_addr(struct svm_range_list *svms,
16448ff079bSFelix Kuehling unsigned long addr,
16548ff079bSFelix Kuehling struct svm_range **parent);
1665fb34bd9SAlex Sierra struct kfd_node *svm_range_get_node_by_id(struct svm_range *prange,
1675fb34bd9SAlex Sierra uint32_t gpu_id);
1685fb34bd9SAlex Sierra int svm_range_vram_node_new(struct kfd_node *node, struct svm_range *prange,
1695fb34bd9SAlex Sierra bool clear);
170e49fe404SFelix Kuehling void svm_range_vram_node_free(struct svm_range *prange);
17148ff079bSFelix Kuehling int svm_range_split_by_granularity(struct kfd_process *p, struct mm_struct *mm,
17248ff079bSFelix Kuehling unsigned long addr, struct svm_range *parent,
17348ff079bSFelix Kuehling struct svm_range *prange);
1745fb34bd9SAlex Sierra int svm_range_restore_pages(struct amdgpu_device *adev, unsigned int pasid,
175f5fe7edfSMukul Joshi uint32_t vmid, uint32_t node_id, uint64_t addr,
1765fb34bd9SAlex Sierra bool write_fault);
177b41896e3SFelix Kuehling int svm_range_schedule_evict_svm_bo(struct amdgpu_amdkfd_fence *fence);
17848ff079bSFelix Kuehling void svm_range_add_list_work(struct svm_range_list *svms,
17948ff079bSFelix Kuehling struct svm_range *prange, struct mm_struct *mm,
18048ff079bSFelix Kuehling enum svm_work_list_ops op);
18148ff079bSFelix Kuehling void schedule_deferred_list_work(struct svm_range_list *svms);
1820b0e518dSFelix Kuehling void svm_range_dma_unmap(struct device *dev, dma_addr_t *dma_addr,
1830b0e518dSFelix Kuehling unsigned long offset, unsigned long npages);
184*ab3400ebSAlex Sierra void svm_range_free_dma_mappings(struct svm_range *prange, bool unmap_dma);
18508a987a8SRajneesh Bhardwaj int svm_range_get_info(struct kfd_process *p, uint32_t *num_svm_ranges,
18608a987a8SRajneesh Bhardwaj uint64_t *svm_priv_data_size);
1879d5dabfeSRajneesh Bhardwaj int kfd_criu_checkpoint_svm(struct kfd_process *p,
1889d5dabfeSRajneesh Bhardwaj uint8_t __user *user_priv_data,
1899d5dabfeSRajneesh Bhardwaj uint64_t *priv_offset);
190c2db32ceSRajneesh Bhardwaj int kfd_criu_restore_svm(struct kfd_process *p,
191c2db32ceSRajneesh Bhardwaj uint8_t __user *user_priv_ptr,
192c2db32ceSRajneesh Bhardwaj uint64_t *priv_data_offset,
193c2db32ceSRajneesh Bhardwaj uint64_t max_priv_data_size);
1942a909ae7SRajneesh Bhardwaj int kfd_criu_resume_svm(struct kfd_process *p);
195d4ebc200SPhilip Yang struct kfd_process_device *
1965fb34bd9SAlex Sierra svm_range_get_pdd_by_node(struct svm_range *prange, struct kfd_node *node);
1976bdfc37bSYifan Zhang void svm_range_list_lock_and_flush_work(struct svm_range_list *svms, struct mm_struct *mm);
19842de677fSPhilip Yang
1995a75ea56SFelix Kuehling /* SVM API and HMM page migration work together, device memory type
2005a75ea56SFelix Kuehling * is initialized to not 0 when page migration register device memory.
2015a75ea56SFelix Kuehling */
202610dab11SPhilip Yang #define KFD_IS_SVM_API_SUPPORTED(adev) ((adev)->kfd.pgmap.type != 0 ||\
203610dab11SPhilip Yang (adev)->gmc.is_app_apu)
2045a75ea56SFelix Kuehling
20569879b30SPhilip Yang void svm_range_bo_unref_async(struct svm_range_bo *svm_bo);
2064959e609SPhilip Yang
2074959e609SPhilip Yang void svm_range_set_max_pages(struct amdgpu_device *adev);
2088a7c3ce1SPhilip Yang int svm_range_switch_xnack_reserve_mem(struct kfd_process *p, bool xnack_enabled);
2094959e609SPhilip Yang
2104ab159d2SFelix Kuehling #else
2114ab159d2SFelix Kuehling
2124ab159d2SFelix Kuehling struct kfd_process;
2134ab159d2SFelix Kuehling
svm_range_list_init(struct kfd_process * p)2144ab159d2SFelix Kuehling static inline int svm_range_list_init(struct kfd_process *p)
2154ab159d2SFelix Kuehling {
2164ab159d2SFelix Kuehling return 0;
2174ab159d2SFelix Kuehling }
svm_range_list_fini(struct kfd_process * p)2184ab159d2SFelix Kuehling static inline void svm_range_list_fini(struct kfd_process *p)
2194ab159d2SFelix Kuehling {
2204ab159d2SFelix Kuehling /* empty */
2214ab159d2SFelix Kuehling }
2224ab159d2SFelix Kuehling
svm_range_restore_pages(struct amdgpu_device * adev,unsigned int pasid,uint32_t client_id,uint32_t node_id,uint64_t addr,bool write_fault)2234ab159d2SFelix Kuehling static inline int svm_range_restore_pages(struct amdgpu_device *adev,
2245fb34bd9SAlex Sierra unsigned int pasid,
2255fb34bd9SAlex Sierra uint32_t client_id, uint32_t node_id,
2265fb34bd9SAlex Sierra uint64_t addr, bool write_fault)
2274ab159d2SFelix Kuehling {
2284ab159d2SFelix Kuehling return -EFAULT;
2294ab159d2SFelix Kuehling }
2304ab159d2SFelix Kuehling
svm_range_schedule_evict_svm_bo(struct amdgpu_amdkfd_fence * fence)2314ab159d2SFelix Kuehling static inline int svm_range_schedule_evict_svm_bo(
2324ab159d2SFelix Kuehling struct amdgpu_amdkfd_fence *fence)
2334ab159d2SFelix Kuehling {
2344ab159d2SFelix Kuehling WARN_ONCE(1, "SVM eviction fence triggered, but SVM is disabled");
2354ab159d2SFelix Kuehling return -EINVAL;
2364ab159d2SFelix Kuehling }
2374ab159d2SFelix Kuehling
svm_range_get_info(struct kfd_process * p,uint32_t * num_svm_ranges,uint64_t * svm_priv_data_size)23808a987a8SRajneesh Bhardwaj static inline int svm_range_get_info(struct kfd_process *p,
23908a987a8SRajneesh Bhardwaj uint32_t *num_svm_ranges,
24008a987a8SRajneesh Bhardwaj uint64_t *svm_priv_data_size)
24108a987a8SRajneesh Bhardwaj {
24208a987a8SRajneesh Bhardwaj *num_svm_ranges = 0;
24308a987a8SRajneesh Bhardwaj *svm_priv_data_size = 0;
24408a987a8SRajneesh Bhardwaj return 0;
24508a987a8SRajneesh Bhardwaj }
24608a987a8SRajneesh Bhardwaj
kfd_criu_checkpoint_svm(struct kfd_process * p,uint8_t __user * user_priv_data,uint64_t * priv_offset)2479d5dabfeSRajneesh Bhardwaj static inline int kfd_criu_checkpoint_svm(struct kfd_process *p,
2489d5dabfeSRajneesh Bhardwaj uint8_t __user *user_priv_data,
2499d5dabfeSRajneesh Bhardwaj uint64_t *priv_offset)
2509d5dabfeSRajneesh Bhardwaj {
2519d5dabfeSRajneesh Bhardwaj return 0;
2529d5dabfeSRajneesh Bhardwaj }
2539d5dabfeSRajneesh Bhardwaj
kfd_criu_restore_svm(struct kfd_process * p,uint8_t __user * user_priv_ptr,uint64_t * priv_data_offset,uint64_t max_priv_data_size)254c2db32ceSRajneesh Bhardwaj static inline int kfd_criu_restore_svm(struct kfd_process *p,
255c2db32ceSRajneesh Bhardwaj uint8_t __user *user_priv_ptr,
256c2db32ceSRajneesh Bhardwaj uint64_t *priv_data_offset,
257c2db32ceSRajneesh Bhardwaj uint64_t max_priv_data_size)
258c2db32ceSRajneesh Bhardwaj {
259c2db32ceSRajneesh Bhardwaj return -EINVAL;
260c2db32ceSRajneesh Bhardwaj }
261c2db32ceSRajneesh Bhardwaj
kfd_criu_resume_svm(struct kfd_process * p)2622a909ae7SRajneesh Bhardwaj static inline int kfd_criu_resume_svm(struct kfd_process *p)
2632a909ae7SRajneesh Bhardwaj {
2642a909ae7SRajneesh Bhardwaj return 0;
2652a909ae7SRajneesh Bhardwaj }
2662a909ae7SRajneesh Bhardwaj
svm_range_set_max_pages(struct amdgpu_device * adev)26784b4dd3fSPhilip Yang static inline void svm_range_set_max_pages(struct amdgpu_device *adev)
26884b4dd3fSPhilip Yang {
26984b4dd3fSPhilip Yang }
27084b4dd3fSPhilip Yang
2715a75ea56SFelix Kuehling #define KFD_IS_SVM_API_SUPPORTED(dev) false
2725a75ea56SFelix Kuehling
2734ab159d2SFelix Kuehling #endif /* IS_ENABLED(CONFIG_HSA_AMD_SVM) */
2744ab159d2SFelix Kuehling
27542de677fSPhilip Yang #endif /* KFD_SVM_H_ */
276