xref: /openbmc/qemu/migration/multifd-qpl.c (revision 05adb38839ba656c7383a548b460d95c91e2febe)
1  /*
2   * Multifd qpl compression accelerator implementation
3   *
4   * Copyright (c) 2023 Intel Corporation
5   *
6   * Authors:
7   *  Yuan Liu<yuan1.liu@intel.com>
8   *
9   * This work is licensed under the terms of the GNU GPL, version 2 or later.
10   * See the COPYING file in the top-level directory.
11   */
12  
13  #include "qemu/osdep.h"
14  #include "qemu/module.h"
15  #include "qapi/error.h"
16  #include "qapi/qapi-types-migration.h"
17  #include "exec/ramblock.h"
18  #include "multifd.h"
19  #include "qpl/qpl.h"
20  
21  /* Maximum number of retries to resubmit a job if IAA work queues are full */
22  #define MAX_SUBMIT_RETRY_NUM (3)
23  
24  typedef struct {
25      /* the QPL hardware path job */
26      qpl_job *job;
27      /* indicates if fallback to software path is required */
28      bool fallback_sw_path;
29      /* output data from the software path */
30      uint8_t *sw_output;
31      /* output data length from the software path */
32      uint32_t sw_output_len;
33  } QplHwJob;
34  
35  typedef struct {
36      /* array of hardware jobs, the number of jobs equals the number pages */
37      QplHwJob *hw_jobs;
38      /* the QPL software job for the slow path and software fallback */
39      qpl_job *sw_job;
40      /* the number of pages that the QPL needs to process at one time */
41      uint32_t page_num;
42      /* array of compressed page buffers */
43      uint8_t *zbuf;
44      /* array of compressed page lengths */
45      uint32_t *zlen;
46      /* the status of the hardware device */
47      bool hw_avail;
48  } QplData;
49  
50  /**
51   * check_hw_avail: check if IAA hardware is available
52   *
53   * If the IAA hardware does not exist or is unavailable,
54   * the QPL hardware job initialization will fail.
55   *
56   * Returns true if IAA hardware is available, otherwise false.
57   *
58   * @job_size: indicates the hardware job size if hardware is available
59   */
60  static bool check_hw_avail(uint32_t *job_size)
61  {
62      qpl_path_t path = qpl_path_hardware;
63      uint32_t size = 0;
64      qpl_job *job;
65  
66      if (qpl_get_job_size(path, &size) != QPL_STS_OK) {
67          return false;
68      }
69      assert(size > 0);
70      job = g_malloc0(size);
71      if (qpl_init_job(path, job) != QPL_STS_OK) {
72          g_free(job);
73          return false;
74      }
75      g_free(job);
76      *job_size = size;
77      return true;
78  }
79  
80  /**
81   * multifd_qpl_free_sw_job: clean up software job
82   *
83   * Free the software job resources.
84   *
85   * @qpl: pointer to the QplData structure
86   */
87  static void multifd_qpl_free_sw_job(QplData *qpl)
88  {
89      assert(qpl);
90      if (qpl->sw_job) {
91          qpl_fini_job(qpl->sw_job);
92          g_free(qpl->sw_job);
93          qpl->sw_job = NULL;
94      }
95  }
96  
97  /**
98   * multifd_qpl_free_jobs: clean up hardware jobs
99   *
100   * Free all hardware job resources.
101   *
102   * @qpl: pointer to the QplData structure
103   */
104  static void multifd_qpl_free_hw_job(QplData *qpl)
105  {
106      assert(qpl);
107      if (qpl->hw_jobs) {
108          for (int i = 0; i < qpl->page_num; i++) {
109              qpl_fini_job(qpl->hw_jobs[i].job);
110              g_free(qpl->hw_jobs[i].job);
111              qpl->hw_jobs[i].job = NULL;
112          }
113          g_free(qpl->hw_jobs);
114          qpl->hw_jobs = NULL;
115      }
116  }
117  
118  /**
119   * multifd_qpl_init_sw_job: initialize a software job
120   *
121   * Use the QPL software path to initialize a job
122   *
123   * @qpl: pointer to the QplData structure
124   * @errp: pointer to an error
125   */
126  static int multifd_qpl_init_sw_job(QplData *qpl, Error **errp)
127  {
128      qpl_path_t path = qpl_path_software;
129      uint32_t size = 0;
130      qpl_job *job = NULL;
131      qpl_status status;
132  
133      status = qpl_get_job_size(path, &size);
134      if (status != QPL_STS_OK) {
135          error_setg(errp, "qpl_get_job_size failed with error %d", status);
136          return -1;
137      }
138      job = g_malloc0(size);
139      status = qpl_init_job(path, job);
140      if (status != QPL_STS_OK) {
141          error_setg(errp, "qpl_init_job failed with error %d", status);
142          g_free(job);
143          return -1;
144      }
145      qpl->sw_job = job;
146      return 0;
147  }
148  
149  /**
150   * multifd_qpl_init_jobs: initialize hardware jobs
151   *
152   * Use the QPL hardware path to initialize jobs
153   *
154   * @qpl: pointer to the QplData structure
155   * @size: the size of QPL hardware path job
156   * @errp: pointer to an error
157   */
158  static void multifd_qpl_init_hw_job(QplData *qpl, uint32_t size, Error **errp)
159  {
160      qpl_path_t path = qpl_path_hardware;
161      qpl_job *job = NULL;
162      qpl_status status;
163  
164      qpl->hw_jobs = g_new0(QplHwJob, qpl->page_num);
165      for (int i = 0; i < qpl->page_num; i++) {
166          job = g_malloc0(size);
167          status = qpl_init_job(path, job);
168          /* the job initialization should succeed after check_hw_avail */
169          assert(status == QPL_STS_OK);
170          qpl->hw_jobs[i].job = job;
171      }
172  }
173  
174  /**
175   * multifd_qpl_init: initialize QplData structure
176   *
177   * Allocate and initialize a QplData structure
178   *
179   * Returns a QplData pointer on success or NULL on error
180   *
181   * @num: the number of pages
182   * @size: the page size
183   * @errp: pointer to an error
184   */
185  static QplData *multifd_qpl_init(uint32_t num, uint32_t size, Error **errp)
186  {
187      uint32_t job_size = 0;
188      QplData *qpl;
189  
190      qpl = g_new0(QplData, 1);
191      qpl->page_num = num;
192      if (multifd_qpl_init_sw_job(qpl, errp) != 0) {
193          g_free(qpl);
194          return NULL;
195      }
196      qpl->hw_avail = check_hw_avail(&job_size);
197      if (qpl->hw_avail) {
198          multifd_qpl_init_hw_job(qpl, job_size, errp);
199      }
200      qpl->zbuf = g_malloc0(size * num);
201      qpl->zlen = g_new0(uint32_t, num);
202      return qpl;
203  }
204  
205  /**
206   * multifd_qpl_deinit: clean up QplData structure
207   *
208   * Free jobs, buffers and the QplData structure
209   *
210   * @qpl: pointer to the QplData structure
211   */
212  static void multifd_qpl_deinit(QplData *qpl)
213  {
214      if (qpl) {
215          multifd_qpl_free_sw_job(qpl);
216          multifd_qpl_free_hw_job(qpl);
217          g_free(qpl->zbuf);
218          g_free(qpl->zlen);
219          g_free(qpl);
220      }
221  }
222  
223  static int multifd_qpl_send_setup(MultiFDSendParams *p, Error **errp)
224  {
225      QplData *qpl;
226      uint32_t page_size = multifd_ram_page_size();
227      uint32_t page_count = multifd_ram_page_count();
228  
229      qpl = multifd_qpl_init(page_count, page_size, errp);
230      if (!qpl) {
231          return -1;
232      }
233      p->compress_data = qpl;
234  
235      /*
236       * the page will be compressed independently and sent using an IOV. The
237       * additional two IOVs are used to store packet header and compressed data
238       * length
239       */
240      p->iov = g_new0(struct iovec, page_count + 2);
241      return 0;
242  }
243  
244  static void multifd_qpl_send_cleanup(MultiFDSendParams *p, Error **errp)
245  {
246      multifd_qpl_deinit(p->compress_data);
247      p->compress_data = NULL;
248      g_free(p->iov);
249      p->iov = NULL;
250  }
251  
252  /**
253   * multifd_qpl_prepare_job: prepare the job
254   *
255   * Set the QPL job parameters and properties.
256   *
257   * @job: pointer to the qpl_job structure
258   * @is_compression: indicates compression and decompression
259   * @input: pointer to the input data buffer
260   * @input_len: the length of the input data
261   * @output: pointer to the output data buffer
262   * @output_len: the length of the output data
263   */
264  static void multifd_qpl_prepare_job(qpl_job *job, bool is_compression,
265                                      uint8_t *input, uint32_t input_len,
266                                      uint8_t *output, uint32_t output_len)
267  {
268      job->op = is_compression ? qpl_op_compress : qpl_op_decompress;
269      job->next_in_ptr = input;
270      job->next_out_ptr = output;
271      job->available_in = input_len;
272      job->available_out = output_len;
273      job->flags = QPL_FLAG_FIRST | QPL_FLAG_LAST | QPL_FLAG_OMIT_VERIFY;
274      /* only supports compression level 1 */
275      job->level = 1;
276  }
277  
278  /**
279   * multifd_qpl_prepare_comp_job: prepare the compression job
280   *
281   * Set the compression job parameters and properties.
282   *
283   * @job: pointer to the qpl_job structure
284   * @input: pointer to the input data buffer
285   * @output: pointer to the output data buffer
286   * @size: the page size
287   */
288  static void multifd_qpl_prepare_comp_job(qpl_job *job, uint8_t *input,
289                                           uint8_t *output, uint32_t size)
290  {
291      /*
292       * Set output length to less than the page size to force the job to
293       * fail in case it compresses to a larger size. We'll send that page
294       * without compression and skip the decompression operation on the
295       * destination.
296       */
297      multifd_qpl_prepare_job(job, true, input, size, output, size - 1);
298  }
299  
300  /**
301   * multifd_qpl_prepare_decomp_job: prepare the decompression job
302   *
303   * Set the decompression job parameters and properties.
304   *
305   * @job: pointer to the qpl_job structure
306   * @input: pointer to the input data buffer
307   * @len: the length of the input data
308   * @output: pointer to the output data buffer
309   * @size: the page size
310   */
311  static void multifd_qpl_prepare_decomp_job(qpl_job *job, uint8_t *input,
312                                             uint32_t len, uint8_t *output,
313                                             uint32_t size)
314  {
315      multifd_qpl_prepare_job(job, false, input, len, output, size);
316  }
317  
318  /**
319   * multifd_qpl_fill_iov: fill in the IOV
320   *
321   * Fill in the QPL packet IOV
322   *
323   * @p: Params for the channel being used
324   * @data: pointer to the IOV data
325   * @len: The length of the IOV data
326   */
327  static void multifd_qpl_fill_iov(MultiFDSendParams *p, uint8_t *data,
328                                   uint32_t len)
329  {
330      p->iov[p->iovs_num].iov_base = data;
331      p->iov[p->iovs_num].iov_len = len;
332      p->iovs_num++;
333      p->next_packet_size += len;
334  }
335  
336  /**
337   * multifd_qpl_fill_packet: fill the compressed page into the QPL packet
338   *
339   * Fill the compressed page length and IOV into the QPL packet
340   *
341   * @idx: The index of the compressed length array
342   * @p: Params for the channel being used
343   * @data: pointer to the compressed page buffer
344   * @len: The length of the compressed page
345   */
346  static void multifd_qpl_fill_packet(uint32_t idx, MultiFDSendParams *p,
347                                      uint8_t *data, uint32_t len)
348  {
349      QplData *qpl = p->compress_data;
350  
351      qpl->zlen[idx] = cpu_to_be32(len);
352      multifd_qpl_fill_iov(p, data, len);
353  }
354  
355  /**
356   * multifd_qpl_submit_job: submit a job to the hardware
357   *
358   * Submit a QPL hardware job to the IAA device
359   *
360   * Returns true if the job is submitted successfully, otherwise false.
361   *
362   * @job: pointer to the qpl_job structure
363   */
364  static bool multifd_qpl_submit_job(qpl_job *job)
365  {
366      qpl_status status;
367      uint32_t num = 0;
368  
369  retry:
370      status = qpl_submit_job(job);
371      if (status == QPL_STS_QUEUES_ARE_BUSY_ERR) {
372          if (num < MAX_SUBMIT_RETRY_NUM) {
373              num++;
374              goto retry;
375          }
376      }
377      return (status == QPL_STS_OK);
378  }
379  
380  /**
381   * multifd_qpl_compress_pages_slow_path: compress pages using slow path
382   *
383   * Compress the pages using software. If compression fails, the uncompressed
384   * page will be sent.
385   *
386   * @p: Params for the channel being used
387   */
388  static void multifd_qpl_compress_pages_slow_path(MultiFDSendParams *p)
389  {
390      QplData *qpl = p->compress_data;
391      MultiFDPages_t *pages = &p->data->u.ram;
392      uint32_t size = multifd_ram_page_size();
393      qpl_job *job = qpl->sw_job;
394      uint8_t *zbuf = qpl->zbuf;
395      uint8_t *buf;
396  
397      for (int i = 0; i < pages->normal_num; i++) {
398          buf = pages->block->host + pages->offset[i];
399          multifd_qpl_prepare_comp_job(job, buf, zbuf, size);
400          if (qpl_execute_job(job) == QPL_STS_OK) {
401              multifd_qpl_fill_packet(i, p, zbuf, job->total_out);
402          } else {
403              /* send the uncompressed page */
404              multifd_qpl_fill_packet(i, p, buf, size);
405          }
406          zbuf += size;
407      }
408  }
409  
410  /**
411   * multifd_qpl_compress_pages: compress pages
412   *
413   * Submit the pages to the IAA hardware for compression. If hardware
414   * compression fails, it falls back to software compression. If software
415   * compression also fails, the uncompressed page is sent.
416   *
417   * @p: Params for the channel being used
418   */
419  static void multifd_qpl_compress_pages(MultiFDSendParams *p)
420  {
421      QplData *qpl = p->compress_data;
422      MultiFDPages_t *pages = &p->data->u.ram;
423      uint32_t size = multifd_ram_page_size();
424      QplHwJob *hw_job;
425      uint8_t *buf;
426      uint8_t *zbuf;
427  
428      for (int i = 0; i < pages->normal_num; i++) {
429          buf = pages->block->host + pages->offset[i];
430          zbuf = qpl->zbuf + (size * i);
431          hw_job = &qpl->hw_jobs[i];
432          multifd_qpl_prepare_comp_job(hw_job->job, buf, zbuf, size);
433          if (multifd_qpl_submit_job(hw_job->job)) {
434              hw_job->fallback_sw_path = false;
435          } else {
436              /*
437               * The IAA work queue is full, any immediate subsequent job
438               * submission is likely to fail, sending the page via the QPL
439               * software path at this point gives us a better chance of
440               * finding the queue open for the next pages.
441               */
442              hw_job->fallback_sw_path = true;
443              multifd_qpl_prepare_comp_job(qpl->sw_job, buf, zbuf, size);
444              if (qpl_execute_job(qpl->sw_job) == QPL_STS_OK) {
445                  hw_job->sw_output = zbuf;
446                  hw_job->sw_output_len = qpl->sw_job->total_out;
447              } else {
448                  hw_job->sw_output = buf;
449                  hw_job->sw_output_len = size;
450              }
451          }
452      }
453  
454      for (int i = 0; i < pages->normal_num; i++) {
455          buf = pages->block->host + pages->offset[i];
456          zbuf = qpl->zbuf + (size * i);
457          hw_job = &qpl->hw_jobs[i];
458          if (hw_job->fallback_sw_path) {
459              multifd_qpl_fill_packet(i, p, hw_job->sw_output,
460                                      hw_job->sw_output_len);
461              continue;
462          }
463          if (qpl_wait_job(hw_job->job) == QPL_STS_OK) {
464              multifd_qpl_fill_packet(i, p, zbuf, hw_job->job->total_out);
465          } else {
466              /* send the uncompressed page */
467              multifd_qpl_fill_packet(i, p, buf, size);
468          }
469      }
470  }
471  
472  static int multifd_qpl_send_prepare(MultiFDSendParams *p, Error **errp)
473  {
474      QplData *qpl = p->compress_data;
475      MultiFDPages_t *pages = &p->data->u.ram;
476      uint32_t len = 0;
477  
478      if (!multifd_send_prepare_common(p)) {
479          goto out;
480      }
481  
482      /* The first IOV is used to store the compressed page lengths */
483      len = pages->normal_num * sizeof(uint32_t);
484      multifd_qpl_fill_iov(p, (uint8_t *) qpl->zlen, len);
485      if (qpl->hw_avail) {
486          multifd_qpl_compress_pages(p);
487      } else {
488          multifd_qpl_compress_pages_slow_path(p);
489      }
490  
491  out:
492      p->flags |= MULTIFD_FLAG_QPL;
493      multifd_send_fill_packet(p);
494      return 0;
495  }
496  
497  static int multifd_qpl_recv_setup(MultiFDRecvParams *p, Error **errp)
498  {
499      QplData *qpl;
500      uint32_t page_size = multifd_ram_page_size();
501      uint32_t page_count = multifd_ram_page_count();
502  
503      qpl = multifd_qpl_init(page_count, page_size, errp);
504      if (!qpl) {
505          return -1;
506      }
507      p->compress_data = qpl;
508      return 0;
509  }
510  
511  static void multifd_qpl_recv_cleanup(MultiFDRecvParams *p)
512  {
513      multifd_qpl_deinit(p->compress_data);
514      p->compress_data = NULL;
515  }
516  
517  /**
518   * multifd_qpl_process_and_check_job: process and check a QPL job
519   *
520   * Process the job and check whether the job output length is the
521   * same as the specified length
522   *
523   * Returns true if the job execution succeeded and the output length
524   * is equal to the specified length, otherwise false.
525   *
526   * @job: pointer to the qpl_job structure
527   * @is_hardware: indicates whether the job is a hardware job
528   * @len: Specified output length
529   * @errp: pointer to an error
530   */
531  static bool multifd_qpl_process_and_check_job(qpl_job *job, bool is_hardware,
532                                                uint32_t len, Error **errp)
533  {
534      qpl_status status;
535  
536      status = (is_hardware ? qpl_wait_job(job) : qpl_execute_job(job));
537      if (status != QPL_STS_OK) {
538          error_setg(errp, "qpl job failed with error %d", status);
539          return false;
540      }
541      if (job->total_out != len) {
542          error_setg(errp, "qpl decompressed len %u, expected len %u",
543                     job->total_out, len);
544          return false;
545      }
546      return true;
547  }
548  
549  /**
550   * multifd_qpl_decompress_pages_slow_path: decompress pages using slow path
551   *
552   * Decompress the pages using software
553   *
554   * Returns 0 on success or -1 on error
555   *
556   * @p: Params for the channel being used
557   * @errp: pointer to an error
558   */
559  static int multifd_qpl_decompress_pages_slow_path(MultiFDRecvParams *p,
560                                                    Error **errp)
561  {
562      QplData *qpl = p->compress_data;
563      uint32_t size = multifd_ram_page_size();
564      qpl_job *job = qpl->sw_job;
565      uint8_t *zbuf = qpl->zbuf;
566      uint8_t *addr;
567      uint32_t len;
568  
569      for (int i = 0; i < p->normal_num; i++) {
570          len = qpl->zlen[i];
571          addr = p->host + p->normal[i];
572          /* the page is uncompressed, load it */
573          if (len == size) {
574              memcpy(addr, zbuf, size);
575              zbuf += size;
576              continue;
577          }
578          multifd_qpl_prepare_decomp_job(job, zbuf, len, addr, size);
579          if (!multifd_qpl_process_and_check_job(job, false, size, errp)) {
580              return -1;
581          }
582          zbuf += len;
583      }
584      return 0;
585  }
586  
587  /**
588   * multifd_qpl_decompress_pages: decompress pages
589   *
590   * Decompress the pages using the IAA hardware. If hardware
591   * decompression fails, it falls back to software decompression.
592   *
593   * Returns 0 on success or -1 on error
594   *
595   * @p: Params for the channel being used
596   * @errp: pointer to an error
597   */
598  static int multifd_qpl_decompress_pages(MultiFDRecvParams *p, Error **errp)
599  {
600      QplData *qpl = p->compress_data;
601      uint32_t size = multifd_ram_page_size();
602      uint8_t *zbuf = qpl->zbuf;
603      uint8_t *addr;
604      uint32_t len;
605      qpl_job *job;
606  
607      for (int i = 0; i < p->normal_num; i++) {
608          addr = p->host + p->normal[i];
609          len = qpl->zlen[i];
610          /* the page is uncompressed if received length equals the page size */
611          if (len == size) {
612              memcpy(addr, zbuf, size);
613              zbuf += size;
614              continue;
615          }
616  
617          job = qpl->hw_jobs[i].job;
618          multifd_qpl_prepare_decomp_job(job, zbuf, len, addr, size);
619          if (multifd_qpl_submit_job(job)) {
620              qpl->hw_jobs[i].fallback_sw_path = false;
621          } else {
622              /*
623               * The IAA work queue is full, any immediate subsequent job
624               * submission is likely to fail, sending the page via the QPL
625               * software path at this point gives us a better chance of
626               * finding the queue open for the next pages.
627               */
628              qpl->hw_jobs[i].fallback_sw_path = true;
629              job = qpl->sw_job;
630              multifd_qpl_prepare_decomp_job(job, zbuf, len, addr, size);
631              if (!multifd_qpl_process_and_check_job(job, false, size, errp)) {
632                  return -1;
633              }
634          }
635          zbuf += len;
636      }
637  
638      for (int i = 0; i < p->normal_num; i++) {
639          /* ignore pages that have already been processed */
640          if (qpl->zlen[i] == size || qpl->hw_jobs[i].fallback_sw_path) {
641              continue;
642          }
643  
644          job = qpl->hw_jobs[i].job;
645          if (!multifd_qpl_process_and_check_job(job, true, size, errp)) {
646              return -1;
647          }
648      }
649      return 0;
650  }
651  static int multifd_qpl_recv(MultiFDRecvParams *p, Error **errp)
652  {
653      QplData *qpl = p->compress_data;
654      uint32_t in_size = p->next_packet_size;
655      uint32_t flags = p->flags & MULTIFD_FLAG_COMPRESSION_MASK;
656      uint32_t len = 0;
657      uint32_t zbuf_len = 0;
658      int ret;
659  
660      if (flags != MULTIFD_FLAG_QPL) {
661          error_setg(errp, "multifd %u: flags received %x flags expected %x",
662                     p->id, flags, MULTIFD_FLAG_QPL);
663          return -1;
664      }
665      multifd_recv_zero_page_process(p);
666      if (!p->normal_num) {
667          assert(in_size == 0);
668          return 0;
669      }
670  
671      /* read compressed page lengths */
672      len = p->normal_num * sizeof(uint32_t);
673      assert(len < in_size);
674      ret = qio_channel_read_all(p->c, (void *) qpl->zlen, len, errp);
675      if (ret != 0) {
676          return ret;
677      }
678      for (int i = 0; i < p->normal_num; i++) {
679          qpl->zlen[i] = be32_to_cpu(qpl->zlen[i]);
680          assert(qpl->zlen[i] <= multifd_ram_page_size());
681          zbuf_len += qpl->zlen[i];
682      }
683  
684      /* read compressed pages */
685      assert(in_size == len + zbuf_len);
686      ret = qio_channel_read_all(p->c, (void *) qpl->zbuf, zbuf_len, errp);
687      if (ret != 0) {
688          return ret;
689      }
690  
691      if (qpl->hw_avail) {
692          return multifd_qpl_decompress_pages(p, errp);
693      }
694      return multifd_qpl_decompress_pages_slow_path(p, errp);
695  }
696  
697  static const MultiFDMethods multifd_qpl_ops = {
698      .send_setup = multifd_qpl_send_setup,
699      .send_cleanup = multifd_qpl_send_cleanup,
700      .send_prepare = multifd_qpl_send_prepare,
701      .recv_setup = multifd_qpl_recv_setup,
702      .recv_cleanup = multifd_qpl_recv_cleanup,
703      .recv = multifd_qpl_recv,
704  };
705  
706  static void multifd_qpl_register(void)
707  {
708      multifd_register_ops(MULTIFD_COMPRESSION_QPL, &multifd_qpl_ops);
709  }
710  
711  migration_init(multifd_qpl_register);
712