xref: /openbmc/linux/drivers/vfio/pci/mlx5/cmd.h (revision f4f0c25e)
1 /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
2 /*
3  * Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
4  */
5 
6 #ifndef MLX5_VFIO_CMD_H
7 #define MLX5_VFIO_CMD_H
8 
9 #include <linux/kernel.h>
10 #include <linux/vfio_pci_core.h>
11 #include <linux/mlx5/driver.h>
12 #include <linux/mlx5/vport.h>
13 #include <linux/mlx5/cq.h>
14 #include <linux/mlx5/qp.h>
15 
16 #define MLX5VF_PRE_COPY_SUPP(mvdev) \
17 	((mvdev)->core_device.vdev.migration_flags & VFIO_MIGRATION_PRE_COPY)
18 
19 enum mlx5_vf_migf_state {
20 	MLX5_MIGF_STATE_ERROR = 1,
21 	MLX5_MIGF_STATE_PRE_COPY_ERROR,
22 	MLX5_MIGF_STATE_PRE_COPY,
23 	MLX5_MIGF_STATE_SAVE_LAST,
24 	MLX5_MIGF_STATE_COMPLETE,
25 };
26 
27 enum mlx5_vf_load_state {
28 	MLX5_VF_LOAD_STATE_READ_IMAGE_NO_HEADER,
29 	MLX5_VF_LOAD_STATE_READ_HEADER,
30 	MLX5_VF_LOAD_STATE_PREP_HEADER_DATA,
31 	MLX5_VF_LOAD_STATE_READ_HEADER_DATA,
32 	MLX5_VF_LOAD_STATE_PREP_IMAGE,
33 	MLX5_VF_LOAD_STATE_READ_IMAGE,
34 	MLX5_VF_LOAD_STATE_LOAD_IMAGE,
35 };
36 
37 struct mlx5_vf_migration_tag_stop_copy_data {
38 	__le64 stop_copy_size;
39 };
40 
41 enum mlx5_vf_migf_header_flags {
42 	MLX5_MIGF_HEADER_FLAGS_TAG_MANDATORY = 0,
43 	MLX5_MIGF_HEADER_FLAGS_TAG_OPTIONAL = 1 << 0,
44 };
45 
46 enum mlx5_vf_migf_header_tag {
47 	MLX5_MIGF_HEADER_TAG_FW_DATA = 0,
48 	MLX5_MIGF_HEADER_TAG_STOP_COPY_SIZE = 1 << 0,
49 };
50 
51 struct mlx5_vf_migration_header {
52 	__le64 record_size;
53 	/* For future use in case we may need to change the kernel protocol */
54 	__le32 flags; /* Use mlx5_vf_migf_header_flags */
55 	__le32 tag; /* Use mlx5_vf_migf_header_tag */
56 	__u8 data[]; /* Its size is given in the record_size */
57 };
58 
59 struct mlx5_vhca_data_buffer {
60 	struct sg_append_table table;
61 	loff_t start_pos;
62 	u64 length;
63 	u64 allocated_length;
64 	u32 mkey;
65 	enum dma_data_direction dma_dir;
66 	u8 dmaed:1;
67 	struct list_head buf_elm;
68 	struct mlx5_vf_migration_file *migf;
69 	/* Optimize mlx5vf_get_migration_page() for sequential access */
70 	struct scatterlist *last_offset_sg;
71 	unsigned int sg_last_entry;
72 	unsigned long last_offset;
73 };
74 
75 struct mlx5vf_async_data {
76 	struct mlx5_async_work cb_work;
77 	struct work_struct work;
78 	struct mlx5_vhca_data_buffer *buf;
79 	struct mlx5_vhca_data_buffer *header_buf;
80 	int status;
81 	u8 last_chunk:1;
82 	void *out;
83 };
84 
85 struct mlx5_vf_migration_file {
86 	struct file *filp;
87 	struct mutex lock;
88 	enum mlx5_vf_migf_state state;
89 
90 	enum mlx5_vf_load_state load_state;
91 	u32 pdn;
92 	loff_t max_pos;
93 	u64 record_size;
94 	u32 record_tag;
95 	u64 stop_copy_prep_size;
96 	u64 pre_copy_initial_bytes;
97 	struct mlx5_vhca_data_buffer *buf;
98 	struct mlx5_vhca_data_buffer *buf_header;
99 	spinlock_t list_lock;
100 	struct list_head buf_list;
101 	struct list_head avail_list;
102 	struct mlx5vf_pci_core_device *mvdev;
103 	wait_queue_head_t poll_wait;
104 	struct completion save_comp;
105 	struct mlx5_async_ctx async_ctx;
106 	struct mlx5vf_async_data async_data;
107 };
108 
109 struct mlx5_vhca_cq_buf {
110 	struct mlx5_frag_buf_ctrl fbc;
111 	struct mlx5_frag_buf frag_buf;
112 	int cqe_size;
113 	int nent;
114 };
115 
116 struct mlx5_vhca_cq {
117 	struct mlx5_vhca_cq_buf buf;
118 	struct mlx5_db db;
119 	struct mlx5_core_cq mcq;
120 	size_t ncqe;
121 };
122 
123 struct mlx5_vhca_recv_buf {
124 	u32 npages;
125 	struct page **page_list;
126 	dma_addr_t *dma_addrs;
127 	u32 next_rq_offset;
128 	u32 mkey;
129 };
130 
131 struct mlx5_vhca_qp {
132 	struct mlx5_frag_buf buf;
133 	struct mlx5_db db;
134 	struct mlx5_vhca_recv_buf recv_buf;
135 	u32 tracked_page_size;
136 	u32 max_msg_size;
137 	u32 qpn;
138 	struct {
139 		unsigned int pc;
140 		unsigned int cc;
141 		unsigned int wqe_cnt;
142 		__be32 *db;
143 		struct mlx5_frag_buf_ctrl fbc;
144 	} rq;
145 };
146 
147 struct mlx5_vhca_page_tracker {
148 	u32 id;
149 	u32 pdn;
150 	u8 is_err:1;
151 	struct mlx5_uars_page *uar;
152 	struct mlx5_vhca_cq cq;
153 	struct mlx5_vhca_qp *host_qp;
154 	struct mlx5_vhca_qp *fw_qp;
155 	struct mlx5_nb nb;
156 	int status;
157 };
158 
159 struct mlx5vf_pci_core_device {
160 	struct vfio_pci_core_device core_device;
161 	int vf_id;
162 	u16 vhca_id;
163 	u8 migrate_cap:1;
164 	u8 deferred_reset:1;
165 	u8 mdev_detach:1;
166 	u8 log_active:1;
167 	struct completion tracker_comp;
168 	/* protect migration state */
169 	struct mutex state_mutex;
170 	enum vfio_device_mig_state mig_state;
171 	/* protect the reset_done flow */
172 	spinlock_t reset_lock;
173 	struct mlx5_vf_migration_file *resuming_migf;
174 	struct mlx5_vf_migration_file *saving_migf;
175 	struct mlx5_vhca_page_tracker tracker;
176 	struct workqueue_struct *cb_wq;
177 	struct notifier_block nb;
178 	struct mlx5_core_dev *mdev;
179 };
180 
181 enum {
182 	MLX5VF_QUERY_INC = (1UL << 0),
183 	MLX5VF_QUERY_FINAL = (1UL << 1),
184 };
185 
186 int mlx5vf_cmd_suspend_vhca(struct mlx5vf_pci_core_device *mvdev, u16 op_mod);
187 int mlx5vf_cmd_resume_vhca(struct mlx5vf_pci_core_device *mvdev, u16 op_mod);
188 int mlx5vf_cmd_query_vhca_migration_state(struct mlx5vf_pci_core_device *mvdev,
189 					  size_t *state_size, u8 query_flags);
190 void mlx5vf_cmd_set_migratable(struct mlx5vf_pci_core_device *mvdev,
191 			       const struct vfio_migration_ops *mig_ops,
192 			       const struct vfio_log_ops *log_ops);
193 void mlx5vf_cmd_remove_migratable(struct mlx5vf_pci_core_device *mvdev);
194 void mlx5vf_cmd_close_migratable(struct mlx5vf_pci_core_device *mvdev);
195 int mlx5vf_cmd_save_vhca_state(struct mlx5vf_pci_core_device *mvdev,
196 			       struct mlx5_vf_migration_file *migf,
197 			       struct mlx5_vhca_data_buffer *buf, bool inc,
198 			       bool track);
199 int mlx5vf_cmd_load_vhca_state(struct mlx5vf_pci_core_device *mvdev,
200 			       struct mlx5_vf_migration_file *migf,
201 			       struct mlx5_vhca_data_buffer *buf);
202 int mlx5vf_cmd_alloc_pd(struct mlx5_vf_migration_file *migf);
203 void mlx5vf_cmd_dealloc_pd(struct mlx5_vf_migration_file *migf);
204 void mlx5fv_cmd_clean_migf_resources(struct mlx5_vf_migration_file *migf);
205 struct mlx5_vhca_data_buffer *
206 mlx5vf_alloc_data_buffer(struct mlx5_vf_migration_file *migf,
207 			 size_t length, enum dma_data_direction dma_dir);
208 void mlx5vf_free_data_buffer(struct mlx5_vhca_data_buffer *buf);
209 struct mlx5_vhca_data_buffer *
210 mlx5vf_get_data_buffer(struct mlx5_vf_migration_file *migf,
211 		       size_t length, enum dma_data_direction dma_dir);
212 void mlx5vf_put_data_buffer(struct mlx5_vhca_data_buffer *buf);
213 int mlx5vf_add_migration_pages(struct mlx5_vhca_data_buffer *buf,
214 			       unsigned int npages);
215 struct page *mlx5vf_get_migration_page(struct mlx5_vhca_data_buffer *buf,
216 				       unsigned long offset);
217 void mlx5vf_state_mutex_unlock(struct mlx5vf_pci_core_device *mvdev);
218 void mlx5vf_disable_fds(struct mlx5vf_pci_core_device *mvdev);
219 void mlx5vf_mig_file_cleanup_cb(struct work_struct *_work);
220 int mlx5vf_start_page_tracker(struct vfio_device *vdev,
221 		struct rb_root_cached *ranges, u32 nnodes, u64 *page_size);
222 int mlx5vf_stop_page_tracker(struct vfio_device *vdev);
223 int mlx5vf_tracker_read_and_clear(struct vfio_device *vdev, unsigned long iova,
224 			unsigned long length, struct iova_bitmap *dirty);
225 #endif /* MLX5_VFIO_CMD_H */
226