Lines Matching refs:cpa
294 static unsigned long __cpa_addr(struct cpa_data *cpa, unsigned long idx) in __cpa_addr() argument
296 if (cpa->flags & CPA_PAGES_ARRAY) { in __cpa_addr()
297 struct page *page = cpa->pages[idx]; in __cpa_addr()
305 if (cpa->flags & CPA_ARRAY) in __cpa_addr()
306 return cpa->vaddr[idx]; in __cpa_addr()
308 return *cpa->vaddr + idx * PAGE_SIZE; in __cpa_addr()
392 struct cpa_data *cpa = data; in __cpa_flush_tlb() local
395 for (i = 0; i < cpa->numpages; i++) in __cpa_flush_tlb()
396 flush_tlb_one_kernel(fix_addr(__cpa_addr(cpa, i))); in __cpa_flush_tlb()
401 struct cpa_data *cpa = data; in cpa_flush() local
411 if (cpa->force_flush_all || cpa->numpages > tlb_single_page_flush_ceiling) in cpa_flush()
414 on_each_cpu(__cpa_flush_tlb, cpa, 1); in cpa_flush()
420 for (i = 0; i < cpa->numpages; i++) { in cpa_flush()
421 unsigned long addr = __cpa_addr(cpa, i); in cpa_flush()
749 static pte_t *_lookup_address_cpa(struct cpa_data *cpa, unsigned long address, in _lookup_address_cpa() argument
754 if (!cpa->pgd) in _lookup_address_cpa()
757 pgd = cpa->pgd + pgd_index(address); in _lookup_address_cpa()
877 struct cpa_data *cpa) in __should_split_large_page() argument
889 tmp = _lookup_address_cpa(cpa, address, &level, &nx, &rw); in __should_split_large_page()
917 if (numpages < cpa->numpages) in __should_split_large_page()
918 cpa->numpages = numpages; in __should_split_large_page()
929 pgprot_val(req_prot) &= ~pgprot_val(cpa->mask_clr); in __should_split_large_page()
930 pgprot_val(req_prot) |= pgprot_val(cpa->mask_set); in __should_split_large_page()
947 cpa->pfn = pfn; in __should_split_large_page()
969 cpa->force_static_prot = 1; in __should_split_large_page()
990 if (address != lpaddr || cpa->numpages != numpages) in __should_split_large_page()
1018 cpa->flags |= CPA_FLUSHTLB; in __should_split_large_page()
1024 struct cpa_data *cpa) in should_split_large_page() argument
1028 if (cpa->force_split) in should_split_large_page()
1032 do_split = __should_split_large_page(kpte, address, cpa); in should_split_large_page()
1038 static void split_set_pte(struct cpa_data *cpa, pte_t *pte, unsigned long pfn, in split_set_pte() argument
1049 if (!cpa->force_static_prot) in split_set_pte()
1075 __split_large_page(struct cpa_data *cpa, pte_t *kpte, unsigned long address, in __split_large_page() argument
1090 tmp = _lookup_address_cpa(cpa, address, &level, &nx, &rw); in __split_large_page()
1138 split_set_pte(cpa, pbase + i, pfn, ref_prot, lpaddr, lpinc); in __split_large_page()
1180 static int split_large_page(struct cpa_data *cpa, pte_t *kpte, in split_large_page() argument
1193 if (__split_large_page(cpa, kpte, address, base)) in split_large_page()
1356 static void populate_pte(struct cpa_data *cpa, in populate_pte() argument
1367 set_pte(pte, pfn_pte(cpa->pfn, pgprot)); in populate_pte()
1370 cpa->pfn++; in populate_pte()
1375 static long populate_pmd(struct cpa_data *cpa, in populate_pmd() argument
1402 populate_pte(cpa, start, pre_end, cur_pages, pmd, pgprot); in populate_pmd()
1426 set_pmd(pmd, pmd_mkhuge(pfn_pmd(cpa->pfn, in populate_pmd()
1430 cpa->pfn += PMD_SIZE >> PAGE_SHIFT; in populate_pmd()
1443 populate_pte(cpa, start, end, num_pages - cur_pages, in populate_pmd()
1449 static int populate_pud(struct cpa_data *cpa, unsigned long start, p4d_t *p4d, in populate_pud() argument
1457 end = start + (cpa->numpages << PAGE_SHIFT); in populate_pud()
1469 cur_pages = min_t(int, (int)cpa->numpages, cur_pages); in populate_pud()
1480 cur_pages = populate_pmd(cpa, start, pre_end, cur_pages, in populate_pud()
1489 if (cpa->numpages == cur_pages) in populate_pud()
1499 set_pud(pud, pud_mkhuge(pfn_pud(cpa->pfn, in populate_pud()
1503 cpa->pfn += PUD_SIZE >> PAGE_SHIFT; in populate_pud()
1517 tmp = populate_pmd(cpa, start, end, cpa->numpages - cur_pages, in populate_pud()
1531 static int populate_pgd(struct cpa_data *cpa, unsigned long addr) in populate_pgd() argument
1539 pgd_entry = cpa->pgd + pgd_index(addr); in populate_pgd()
1561 pgprot_val(pgprot) &= ~pgprot_val(cpa->mask_clr); in populate_pgd()
1562 pgprot_val(pgprot) |= pgprot_val(cpa->mask_set); in populate_pgd()
1564 ret = populate_pud(cpa, addr, p4d, pgprot); in populate_pgd()
1572 addr + (cpa->numpages << PAGE_SHIFT)); in populate_pgd()
1576 cpa->numpages = ret; in populate_pgd()
1580 static int __cpa_process_fault(struct cpa_data *cpa, unsigned long vaddr, in __cpa_process_fault() argument
1583 if (cpa->pgd) { in __cpa_process_fault()
1589 return populate_pgd(cpa, vaddr); in __cpa_process_fault()
1596 cpa->numpages = 1; in __cpa_process_fault()
1609 cpa->numpages = 1; in __cpa_process_fault()
1610 cpa->pfn = __pa(vaddr) >> PAGE_SHIFT; in __cpa_process_fault()
1613 } else if (__cpa_pfn_in_highmap(cpa->pfn)) { in __cpa_process_fault()
1619 *cpa->vaddr); in __cpa_process_fault()
1625 static int __change_page_attr(struct cpa_data *cpa, int primary) in __change_page_attr() argument
1633 address = __cpa_addr(cpa, cpa->curpage); in __change_page_attr()
1635 kpte = _lookup_address_cpa(cpa, address, &level, &nx, &rw); in __change_page_attr()
1637 return __cpa_process_fault(cpa, address, primary); in __change_page_attr()
1641 return __cpa_process_fault(cpa, address, primary); in __change_page_attr()
1649 pgprot_val(new_prot) &= ~pgprot_val(cpa->mask_clr); in __change_page_attr()
1650 pgprot_val(new_prot) |= pgprot_val(cpa->mask_set); in __change_page_attr()
1668 cpa->pfn = pfn; in __change_page_attr()
1674 cpa->flags |= CPA_FLUSHTLB; in __change_page_attr()
1676 cpa->numpages = 1; in __change_page_attr()
1684 do_split = should_split_large_page(kpte, address, cpa); in __change_page_attr()
1696 err = split_large_page(cpa, kpte, address); in __change_page_attr()
1703 static int __change_page_attr_set_clr(struct cpa_data *cpa, int primary);
1708 static int cpa_process_alias(struct cpa_data *cpa) in cpa_process_alias() argument
1711 unsigned long laddr = (unsigned long)__va(cpa->pfn << PAGE_SHIFT); in cpa_process_alias()
1715 if (!pfn_range_is_mapped(cpa->pfn, cpa->pfn + 1)) in cpa_process_alias()
1722 vaddr = __cpa_addr(cpa, cpa->curpage); in cpa_process_alias()
1726 alias_cpa = *cpa; in cpa_process_alias()
1737 cpa->force_flush_all = 1; in cpa_process_alias()
1751 __cpa_pfn_in_highmap(cpa->pfn)) { in cpa_process_alias()
1752 unsigned long temp_cpa_vaddr = (cpa->pfn << PAGE_SHIFT) + in cpa_process_alias()
1754 alias_cpa = *cpa; in cpa_process_alias()
1768 cpa->force_flush_all = 1; in cpa_process_alias()
1780 static int __change_page_attr_set_clr(struct cpa_data *cpa, int primary) in __change_page_attr_set_clr() argument
1782 unsigned long numpages = cpa->numpages; in __change_page_attr_set_clr()
1789 if (!(pgprot_val(cpa->mask_set) | pgprot_val(cpa->mask_clr)) && in __change_page_attr_set_clr()
1790 !cpa->force_split) in __change_page_attr_set_clr()
1798 cpa->numpages = rempages; in __change_page_attr_set_clr()
1800 if (cpa->flags & (CPA_ARRAY | CPA_PAGES_ARRAY)) in __change_page_attr_set_clr()
1801 cpa->numpages = 1; in __change_page_attr_set_clr()
1805 ret = __change_page_attr(cpa, primary); in __change_page_attr_set_clr()
1811 if (primary && !(cpa->flags & CPA_NO_CHECK_ALIAS)) { in __change_page_attr_set_clr()
1812 ret = cpa_process_alias(cpa); in __change_page_attr_set_clr()
1822 BUG_ON(cpa->numpages > rempages || !cpa->numpages); in __change_page_attr_set_clr()
1823 rempages -= cpa->numpages; in __change_page_attr_set_clr()
1824 cpa->curpage += cpa->numpages; in __change_page_attr_set_clr()
1829 cpa->numpages = numpages; in __change_page_attr_set_clr()
1838 struct cpa_data cpa; in change_page_attr_set_clr() local
1841 memset(&cpa, 0, sizeof(cpa)); in change_page_attr_set_clr()
1880 cpa.vaddr = addr; in change_page_attr_set_clr()
1881 cpa.pages = pages; in change_page_attr_set_clr()
1882 cpa.numpages = numpages; in change_page_attr_set_clr()
1883 cpa.mask_set = mask_set; in change_page_attr_set_clr()
1884 cpa.mask_clr = mask_clr; in change_page_attr_set_clr()
1885 cpa.flags = in_flag; in change_page_attr_set_clr()
1886 cpa.curpage = 0; in change_page_attr_set_clr()
1887 cpa.force_split = force_split; in change_page_attr_set_clr()
1889 ret = __change_page_attr_set_clr(&cpa, 1); in change_page_attr_set_clr()
1894 if (!(cpa.flags & CPA_FLUSHTLB)) in change_page_attr_set_clr()
1911 cpa_flush(&cpa, cache); in change_page_attr_set_clr()
2172 struct cpa_data cpa; in __set_memory_enc_pgtable() local
2179 memset(&cpa, 0, sizeof(cpa)); in __set_memory_enc_pgtable()
2180 cpa.vaddr = &addr; in __set_memory_enc_pgtable()
2181 cpa.numpages = numpages; in __set_memory_enc_pgtable()
2182 cpa.mask_set = enc ? pgprot_encrypted(empty) : pgprot_decrypted(empty); in __set_memory_enc_pgtable()
2183 cpa.mask_clr = enc ? pgprot_decrypted(empty) : pgprot_encrypted(empty); in __set_memory_enc_pgtable()
2184 cpa.pgd = init_mm.pgd; in __set_memory_enc_pgtable()
2192 cpa_flush(&cpa, x86_platform.guest.enc_cache_flush_required()); in __set_memory_enc_pgtable()
2198 ret = __change_page_attr_set_clr(&cpa, 1); in __set_memory_enc_pgtable()
2207 cpa_flush(&cpa, 0); in __set_memory_enc_pgtable()
2354 struct cpa_data cpa = { .vaddr = &tempaddr, in __set_pages_p() local
2367 return __change_page_attr_set_clr(&cpa, 1); in __set_pages_p()
2373 struct cpa_data cpa = { .vaddr = &tempaddr, in __set_pages_np() local
2386 return __change_page_attr_set_clr(&cpa, 1); in __set_pages_np()
2450 struct cpa_data cpa = { in kernel_map_pages_in_pgd() local
2466 cpa.mask_clr = pgprot_encrypted(cpa.mask_clr); in kernel_map_pages_in_pgd()
2468 cpa.mask_set = __pgprot(_PAGE_PRESENT | page_flags); in kernel_map_pages_in_pgd()
2470 retval = __change_page_attr_set_clr(&cpa, 1); in kernel_map_pages_in_pgd()
2493 struct cpa_data cpa = { in kernel_unmap_pages_in_pgd() local
2505 retval = __change_page_attr_set_clr(&cpa, 1); in kernel_unmap_pages_in_pgd()