1b2441318SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0
21da177e4SLinus Torvalds /*
31da177e4SLinus Torvalds * S390 version
4a53c8fabSHeiko Carstens * Copyright IBM Corp. 1999
51da177e4SLinus Torvalds * Author(s): Hartmut Penner (hp@de.ibm.com)
61da177e4SLinus Torvalds * Ulrich Weigand (uweigand@de.ibm.com)
71da177e4SLinus Torvalds *
81da177e4SLinus Torvalds * Derived from "arch/i386/mm/fault.c"
91da177e4SLinus Torvalds * Copyright (C) 1995 Linus Torvalds
101da177e4SLinus Torvalds */
111da177e4SLinus Torvalds
12052ff461SHeiko Carstens #include <linux/kernel_stat.h>
13cdd6c482SIngo Molnar #include <linux/perf_event.h>
141da177e4SLinus Torvalds #include <linux/signal.h>
151da177e4SLinus Torvalds #include <linux/sched.h>
16b17b0153SIngo Molnar #include <linux/sched/debug.h>
171da177e4SLinus Torvalds #include <linux/kernel.h>
181da177e4SLinus Torvalds #include <linux/errno.h>
191da177e4SLinus Torvalds #include <linux/string.h>
201da177e4SLinus Torvalds #include <linux/types.h>
211da177e4SLinus Torvalds #include <linux/ptrace.h>
221da177e4SLinus Torvalds #include <linux/mman.h>
231da177e4SLinus Torvalds #include <linux/mm.h>
247757591aSHeiko Carstens #include <linux/compat.h>
251da177e4SLinus Torvalds #include <linux/smp.h>
261eeb66a1SChristoph Hellwig #include <linux/kdebug.h>
271da177e4SLinus Torvalds #include <linux/init.h>
281da177e4SLinus Torvalds #include <linux/console.h>
29dcc096c5SPaul Gortmaker #include <linux/extable.h>
301da177e4SLinus Torvalds #include <linux/hardirq.h>
314ba069b8SMichael Grundy #include <linux/kprobes.h>
32be5ec363SMartin Schwidefsky #include <linux/uaccess.h>
3353492b1dSGerald Schaefer #include <linux/hugetlb.h>
34e41ba111SSven Schnelle #include <linux/kfence.h>
35d09a307fSHeiko Carstens #include <asm/asm-extable.h>
36cbb870c8SHeiko Carstens #include <asm/asm-offsets.h>
371ec2772eSMartin Schwidefsky #include <asm/diag.h>
381e133ab2SMartin Schwidefsky #include <asm/gmap.h>
39d7b250e2SHeiko Carstens #include <asm/irq.h>
406252d702SMartin Schwidefsky #include <asm/mmu_context.h>
41a0616cdeSDavid Howells #include <asm/facility.h>
42084ea4d6SVasily Gorbik #include <asm/uv.h>
43a806170eSHeiko Carstens #include "../kernel/entry.h"
441da177e4SLinus Torvalds
451da177e4SLinus Torvalds #define __FAIL_ADDR_MASK -4096L
461da177e4SLinus Torvalds
471da177e4SLinus Torvalds /*
481da177e4SLinus Torvalds * Allocate private vm_fault_reason from top. Please make sure it won't
49d939474bSPeter Xu * collide with vm_fault_reason.
50d939474bSPeter Xu */
51d939474bSPeter Xu #define VM_FAULT_BADCONTEXT ((__force vm_fault_t)0x80000000)
52d939474bSPeter Xu #define VM_FAULT_BADMAP ((__force vm_fault_t)0x40000000)
53d939474bSPeter Xu #define VM_FAULT_BADACCESS ((__force vm_fault_t)0x20000000)
54d939474bSPeter Xu #define VM_FAULT_SIGNAL ((__force vm_fault_t)0x10000000)
55d939474bSPeter Xu #define VM_FAULT_PFAULT ((__force vm_fault_t)0x8000000)
56d939474bSPeter Xu
57d939474bSPeter Xu enum fault_type {
5850d7280dSMartin Schwidefsky KERNEL_FAULT,
590aaba41bSMartin Schwidefsky USER_FAULT,
600aaba41bSMartin Schwidefsky GMAP_FAULT,
610aaba41bSMartin Schwidefsky };
620aaba41bSMartin Schwidefsky
630aaba41bSMartin Schwidefsky static unsigned long store_indication __read_mostly;
640aaba41bSMartin Schwidefsky
fault_init(void)65a4f32bdbSHeiko Carstens static int __init fault_init(void)
6692f842eaSMartin Schwidefsky {
67a4f32bdbSHeiko Carstens if (test_facility(75))
6892f842eaSMartin Schwidefsky store_indication = 0xc00;
69a4f32bdbSHeiko Carstens return 0;
7092f842eaSMartin Schwidefsky }
71a4f32bdbSHeiko Carstens early_initcall(fault_init);
7292f842eaSMartin Schwidefsky
73a4f32bdbSHeiko Carstens /*
7492f842eaSMartin Schwidefsky * Find out which address space caused the exception.
751da177e4SLinus Torvalds */
get_fault_type(struct pt_regs * regs)760aaba41bSMartin Schwidefsky static enum fault_type get_fault_type(struct pt_regs *regs)
771da177e4SLinus Torvalds {
78bf2f1eeeSMasahiro Yamada unsigned long trans_exc_code;
791da177e4SLinus Torvalds
80457f2180SHeiko Carstens trans_exc_code = regs->int_parm_long & 3;
81457f2180SHeiko Carstens if (likely(trans_exc_code == 0)) {
82457f2180SHeiko Carstens /* primary space exception */
830aaba41bSMartin Schwidefsky if (user_mode(regs))
840aaba41bSMartin Schwidefsky return USER_FAULT;
8587d59863SHeiko Carstens if (!IS_ENABLED(CONFIG_PGSTE))
8687d59863SHeiko Carstens return KERNEL_FAULT;
8787d59863SHeiko Carstens if (test_pt_regs_flag(regs, PIF_GUEST_FAULT))
8887d59863SHeiko Carstens return GMAP_FAULT;
8987d59863SHeiko Carstens return KERNEL_FAULT;
900aaba41bSMartin Schwidefsky }
910aaba41bSMartin Schwidefsky if (trans_exc_code == 2)
920aaba41bSMartin Schwidefsky return USER_FAULT;
9387d59863SHeiko Carstens if (trans_exc_code == 1) {
940aaba41bSMartin Schwidefsky /* access register mode, not used in the kernel */
95962f0af8SGerald Schaefer return USER_FAULT;
96962f0af8SGerald Schaefer }
97962f0af8SGerald Schaefer /* home space exception -> access via kernel ASCE */
98962f0af8SGerald Schaefer return KERNEL_FAULT;
990aaba41bSMartin Schwidefsky }
1000aaba41bSMartin Schwidefsky
get_fault_address(struct pt_regs * regs)1011da177e4SLinus Torvalds static unsigned long get_fault_address(struct pt_regs *regs)
1021da177e4SLinus Torvalds {
103d9c2cf67SHeiko Carstens unsigned long trans_exc_code = regs->int_parm_long;
104d9c2cf67SHeiko Carstens
105d9c2cf67SHeiko Carstens return trans_exc_code & __FAIL_ADDR_MASK;
106d9c2cf67SHeiko Carstens }
107d9c2cf67SHeiko Carstens
fault_is_write(struct pt_regs * regs)108d9c2cf67SHeiko Carstens static bool fault_is_write(struct pt_regs *regs)
109d9c2cf67SHeiko Carstens {
110d9c2cf67SHeiko Carstens unsigned long trans_exc_code = regs->int_parm_long;
111d9c2cf67SHeiko Carstens
112d9c2cf67SHeiko Carstens return (trans_exc_code & store_indication) == 0x400;
113d9c2cf67SHeiko Carstens }
114d9c2cf67SHeiko Carstens
bad_address(void * p)115d9c2cf67SHeiko Carstens static int bad_address(void *p)
116d9c2cf67SHeiko Carstens {
1173b7df342SHeiko Carstens unsigned long dummy;
1183b7df342SHeiko Carstens
1193b7df342SHeiko Carstens return get_kernel_nofault(dummy, (unsigned long *)p);
1203b7df342SHeiko Carstens }
12125f12ae4SChristoph Hellwig
dump_pagetable(unsigned long asce,unsigned long address)1223b7df342SHeiko Carstens static void dump_pagetable(unsigned long asce, unsigned long address)
1233b7df342SHeiko Carstens {
1243b7df342SHeiko Carstens unsigned long *table = __va(asce & _ASCE_ORIGIN);
1253b7df342SHeiko Carstens
126fe7b2747SHeiko Carstens pr_alert("AS:%016lx ", asce);
1273b7df342SHeiko Carstens switch (asce & _ASCE_TYPE_MASK) {
1283b7df342SHeiko Carstens case _ASCE_TYPE_REGION1:
1293b7df342SHeiko Carstens table += (address & _REGION1_INDEX) >> _REGION1_SHIFT;
1303b7df342SHeiko Carstens if (bad_address(table))
131f1c1174fSHeiko Carstens goto bad;
1323b7df342SHeiko Carstens pr_cont("R1:%016lx ", *table);
1333b7df342SHeiko Carstens if (*table & _REGION_ENTRY_INVALID)
1343b7df342SHeiko Carstens goto out;
1353b7df342SHeiko Carstens table = __va(*table & _REGION_ENTRY_ORIGIN);
1363b7df342SHeiko Carstens fallthrough;
137d2f2949aSHeiko Carstens case _ASCE_TYPE_REGION2:
1382c7749b9SJoe Perches table += (address & _REGION2_INDEX) >> _REGION2_SHIFT;
1393b7df342SHeiko Carstens if (bad_address(table))
140f1c1174fSHeiko Carstens goto bad;
1413b7df342SHeiko Carstens pr_cont("R2:%016lx ", *table);
1423b7df342SHeiko Carstens if (*table & _REGION_ENTRY_INVALID)
1433b7df342SHeiko Carstens goto out;
1443b7df342SHeiko Carstens table = __va(*table & _REGION_ENTRY_ORIGIN);
1453b7df342SHeiko Carstens fallthrough;
146d2f2949aSHeiko Carstens case _ASCE_TYPE_REGION3:
1472c7749b9SJoe Perches table += (address & _REGION3_INDEX) >> _REGION3_SHIFT;
1483b7df342SHeiko Carstens if (bad_address(table))
149f1c1174fSHeiko Carstens goto bad;
1503b7df342SHeiko Carstens pr_cont("R3:%016lx ", *table);
1513b7df342SHeiko Carstens if (*table & (_REGION_ENTRY_INVALID | _REGION3_ENTRY_LARGE))
1523b7df342SHeiko Carstens goto out;
1533b7df342SHeiko Carstens table = __va(*table & _REGION_ENTRY_ORIGIN);
1543b7df342SHeiko Carstens fallthrough;
155d2f2949aSHeiko Carstens case _ASCE_TYPE_SEGMENT:
1562c7749b9SJoe Perches table += (address & _SEGMENT_INDEX) >> _SEGMENT_SHIFT;
1573b7df342SHeiko Carstens if (bad_address(table))
158f1c1174fSHeiko Carstens goto bad;
1593b7df342SHeiko Carstens pr_cont("S:%016lx ", *table);
1603b7df342SHeiko Carstens if (*table & (_SEGMENT_ENTRY_INVALID | _SEGMENT_ENTRY_LARGE))
16191c0837eSJoe Perches goto out;
1623b7df342SHeiko Carstens table = __va(*table & _SEGMENT_ENTRY_ORIGIN);
1633b7df342SHeiko Carstens }
164d2f2949aSHeiko Carstens table += (address & _PAGE_INDEX) >> _PAGE_SHIFT;
1653b7df342SHeiko Carstens if (bad_address(table))
166f1c1174fSHeiko Carstens goto bad;
1673b7df342SHeiko Carstens pr_cont("P:%016lx ", *table);
1683b7df342SHeiko Carstens out:
1693b7df342SHeiko Carstens pr_cont("\n");
1703b7df342SHeiko Carstens return;
1713b7df342SHeiko Carstens bad:
1723b7df342SHeiko Carstens pr_cont("BAD\n");
1733b7df342SHeiko Carstens }
1743b7df342SHeiko Carstens
dump_fault_info(struct pt_regs * regs)1753b7df342SHeiko Carstens static void dump_fault_info(struct pt_regs *regs)
1763b7df342SHeiko Carstens {
1773b7df342SHeiko Carstens unsigned long asce;
1783b7df342SHeiko Carstens
1793b7df342SHeiko Carstens pr_alert("Failing address: %016lx TEID: %016lx\n",
1803b7df342SHeiko Carstens regs->int_parm_long & __FAIL_ADDR_MASK, regs->int_parm_long);
1815d7eccecSHeiko Carstens pr_alert("Fault in ");
1825d7eccecSHeiko Carstens switch (regs->int_parm_long & 3) {
1833b7df342SHeiko Carstens case 3:
1843b7df342SHeiko Carstens pr_cont("home space ");
1853b7df342SHeiko Carstens break;
1863b7df342SHeiko Carstens case 2:
1873b7df342SHeiko Carstens pr_cont("secondary space ");
1883b7df342SHeiko Carstens break;
1893b7df342SHeiko Carstens case 1:
1903b7df342SHeiko Carstens pr_cont("access register ");
1913b7df342SHeiko Carstens break;
1923b7df342SHeiko Carstens case 0:
1933b7df342SHeiko Carstens pr_cont("primary space ");
1943b7df342SHeiko Carstens break;
1953b7df342SHeiko Carstens }
1963b7df342SHeiko Carstens pr_cont("mode while using ");
1973b7df342SHeiko Carstens switch (get_fault_type(regs)) {
1983b7df342SHeiko Carstens case USER_FAULT:
1990aaba41bSMartin Schwidefsky asce = S390_lowcore.user_asce;
2000aaba41bSMartin Schwidefsky pr_cont("user ");
2013b7df342SHeiko Carstens break;
2023b7df342SHeiko Carstens case GMAP_FAULT:
2030aaba41bSMartin Schwidefsky asce = ((struct gmap *) S390_lowcore.gmap)->asce;
2040aaba41bSMartin Schwidefsky pr_cont("gmap ");
2050aaba41bSMartin Schwidefsky break;
2060aaba41bSMartin Schwidefsky case KERNEL_FAULT:
2070aaba41bSMartin Schwidefsky asce = S390_lowcore.kernel_asce;
2080aaba41bSMartin Schwidefsky pr_cont("kernel ");
2090aaba41bSMartin Schwidefsky break;
2100aaba41bSMartin Schwidefsky default:
2110aaba41bSMartin Schwidefsky unreachable();
212bf2f1eeeSMasahiro Yamada }
213bf2f1eeeSMasahiro Yamada pr_cont("ASCE.\n");
2143b7df342SHeiko Carstens dump_pagetable(asce, regs->int_parm_long & __FAIL_ADDR_MASK);
2153b7df342SHeiko Carstens }
2163b7df342SHeiko Carstens
2173b7df342SHeiko Carstens int show_unhandled_signals = 1;
2183b7df342SHeiko Carstens
report_user_fault(struct pt_regs * regs,long signr,int is_mm_fault)2195d7eccecSHeiko Carstens void report_user_fault(struct pt_regs *regs, long signr, int is_mm_fault)
2205d7eccecSHeiko Carstens {
2215d7eccecSHeiko Carstens if ((task_pid_nr(current) > 1) && !show_unhandled_signals)
222ab3c68eeSHeiko Carstens return;
223ab3c68eeSHeiko Carstens if (!unhandled_signal(current, signr))
224ab3c68eeSHeiko Carstens return;
225ab3c68eeSHeiko Carstens if (!printk_ratelimit())
226ab3c68eeSHeiko Carstens return;
227ab3c68eeSHeiko Carstens printk(KERN_ALERT "User process fault: interruption code %04x ilc:%d ",
228ab3c68eeSHeiko Carstens regs->int_code & 0xffff, regs->int_code >> 17);
229413d4047SHeiko Carstens print_vma_addr(KERN_CONT "in ", regs->psw.addr);
230413d4047SHeiko Carstens printk(KERN_CONT "\n");
2319cb1ccecSHeiko Carstens if (is_mm_fault)
232aa33c8cbSMartin Schwidefsky dump_fault_info(regs);
2335d7eccecSHeiko Carstens show_regs(regs);
2343b7df342SHeiko Carstens }
235ab3c68eeSHeiko Carstens
236ab3c68eeSHeiko Carstens /*
237ab3c68eeSHeiko Carstens * Send SIGSEGV to task. This is an external routine
2381da177e4SLinus Torvalds * to keep the stack usage of do_page_fault small.
2391da177e4SLinus Torvalds */
do_sigsegv(struct pt_regs * regs,int si_code)2401da177e4SLinus Torvalds static noinline void do_sigsegv(struct pt_regs *regs, int si_code)
2411da177e4SLinus Torvalds {
242aa33c8cbSMartin Schwidefsky report_user_fault(regs, SIGSEGV, 1);
2431da177e4SLinus Torvalds force_sig_fault(SIGSEGV, si_code,
2445d7eccecSHeiko Carstens (void __user *)(regs->int_parm_long & __FAIL_ADDR_MASK));
2459507a5d0SEric W. Biederman }
2462e1661d2SEric W. Biederman
do_no_context(struct pt_regs * regs,vm_fault_t fault)2471da177e4SLinus Torvalds static noinline void do_no_context(struct pt_regs *regs, vm_fault_t fault)
2481da177e4SLinus Torvalds {
249d9c2cf67SHeiko Carstens enum fault_type fault_type;
25010c1031fSMartin Schwidefsky unsigned long address;
251d9c2cf67SHeiko Carstens bool is_write;
252d9c2cf67SHeiko Carstens
253d9c2cf67SHeiko Carstens if (fixup_exception(regs))
254d9c2cf67SHeiko Carstens return;
25546fee16fSHeiko Carstens fault_type = get_fault_type(regs);
25610c1031fSMartin Schwidefsky if ((fault_type == KERNEL_FAULT) && (fault == VM_FAULT_BADCONTEXT)) {
257d9c2cf67SHeiko Carstens address = get_fault_address(regs);
258d9c2cf67SHeiko Carstens is_write = fault_is_write(regs);
259d9c2cf67SHeiko Carstens if (kfence_handle_page_fault(address, is_write, regs))
260d9c2cf67SHeiko Carstens return;
261d9c2cf67SHeiko Carstens }
262d9c2cf67SHeiko Carstens /*
263d9c2cf67SHeiko Carstens * Oops. The kernel tried to access some bad page. We'll have to
26410c1031fSMartin Schwidefsky * terminate things with extreme prejudice.
26510c1031fSMartin Schwidefsky */
26610c1031fSMartin Schwidefsky if (fault_type == KERNEL_FAULT)
26710c1031fSMartin Schwidefsky printk(KERN_ALERT "Unable to handle kernel pointer dereference"
268d9c2cf67SHeiko Carstens " in virtual kernel address space\n");
26910c1031fSMartin Schwidefsky else
2703b7df342SHeiko Carstens printk(KERN_ALERT "Unable to handle kernel paging request"
27110c1031fSMartin Schwidefsky " in virtual user address space\n");
27210c1031fSMartin Schwidefsky dump_fault_info(regs);
2733b7df342SHeiko Carstens die(regs, "Oops");
2743b7df342SHeiko Carstens }
275aa33c8cbSMartin Schwidefsky
do_low_address(struct pt_regs * regs)27610c1031fSMartin Schwidefsky static noinline void do_low_address(struct pt_regs *regs)
27710c1031fSMartin Schwidefsky {
278aa33c8cbSMartin Schwidefsky /* Low-address protection hit in kernel mode means
27910c1031fSMartin Schwidefsky NULL pointer write access in kernel mode. */
28010c1031fSMartin Schwidefsky if (regs->psw.mask & PSW_MASK_PSTATE) {
28110c1031fSMartin Schwidefsky /* Low-address protection hit in user mode 'cannot happen'. */
28210c1031fSMartin Schwidefsky die (regs, "Low-address protection");
28310c1031fSMartin Schwidefsky }
284aa33c8cbSMartin Schwidefsky
28510c1031fSMartin Schwidefsky do_no_context(regs, VM_FAULT_BADACCESS);
28610c1031fSMartin Schwidefsky }
287d9c2cf67SHeiko Carstens
do_sigbus(struct pt_regs * regs)28810c1031fSMartin Schwidefsky static noinline void do_sigbus(struct pt_regs *regs)
28910c1031fSMartin Schwidefsky {
290aa33c8cbSMartin Schwidefsky /*
29110c1031fSMartin Schwidefsky * Send a sigbus, regardless of whether we were in kernel
29210c1031fSMartin Schwidefsky * or user mode.
29310c1031fSMartin Schwidefsky */
29410c1031fSMartin Schwidefsky force_sig_fault(SIGBUS, BUS_ADRERR,
29510c1031fSMartin Schwidefsky (void __user *)(regs->int_parm_long & __FAIL_ADDR_MASK));
2969507a5d0SEric W. Biederman }
2972e1661d2SEric W. Biederman
do_fault_error(struct pt_regs * regs,vm_fault_t fault)29810c1031fSMartin Schwidefsky static noinline void do_fault_error(struct pt_regs *regs, vm_fault_t fault)
29910c1031fSMartin Schwidefsky {
300bf2ce385SHeiko Carstens int si_code;
30150d7280dSMartin Schwidefsky
30250d7280dSMartin Schwidefsky switch (fault) {
30350d7280dSMartin Schwidefsky case VM_FAULT_BADACCESS:
30450d7280dSMartin Schwidefsky case VM_FAULT_BADMAP:
30550d7280dSMartin Schwidefsky /* Bad memory access. Check if it is kernel or user space. */
30650d7280dSMartin Schwidefsky if (user_mode(regs)) {
30750d7280dSMartin Schwidefsky /* User mode accesses just cause a SIGSEGV */
3087d256175SHeiko Carstens si_code = (fault == VM_FAULT_BADMAP) ?
30950d7280dSMartin Schwidefsky SEGV_MAPERR : SEGV_ACCERR;
31050d7280dSMartin Schwidefsky do_sigsegv(regs, si_code);
31150d7280dSMartin Schwidefsky break;
312aa33c8cbSMartin Schwidefsky }
31357d7f939SMartin Schwidefsky fallthrough;
31450d7280dSMartin Schwidefsky case VM_FAULT_BADCONTEXT:
3152c7749b9SJoe Perches case VM_FAULT_PFAULT:
31650d7280dSMartin Schwidefsky do_no_context(regs, fault);
31724eb3a82SDominik Dingel break;
318d9c2cf67SHeiko Carstens case VM_FAULT_SIGNAL:
31950d7280dSMartin Schwidefsky if (!user_mode(regs))
320f2c76e3bSHeiko Carstens do_no_context(regs, fault);
321f2c76e3bSHeiko Carstens break;
322d9c2cf67SHeiko Carstens default: /* fault & VM_FAULT_ERROR */
323f2c76e3bSHeiko Carstens if (fault & VM_FAULT_OOM) {
32450d7280dSMartin Schwidefsky if (!user_mode(regs))
32599583181SHeiko Carstens do_no_context(regs, fault);
3267d256175SHeiko Carstens else
327d9c2cf67SHeiko Carstens pagefault_out_of_memory();
32899583181SHeiko Carstens } else if (fault & VM_FAULT_SIGSEGV) {
32950d7280dSMartin Schwidefsky /* Kernel mode? Handle exceptions or die */
33033692f27SLinus Torvalds if (!user_mode(regs))
33133692f27SLinus Torvalds do_no_context(regs, fault);
33233692f27SLinus Torvalds else
333d9c2cf67SHeiko Carstens do_sigsegv(regs, SEGV_MAPERR);
33433692f27SLinus Torvalds } else if (fault & VM_FAULT_SIGBUS) {
33533692f27SLinus Torvalds /* Kernel mode? Handle exceptions or die */
33699583181SHeiko Carstens if (!user_mode(regs))
33750d7280dSMartin Schwidefsky do_no_context(regs, fault);
3387d256175SHeiko Carstens else
339d9c2cf67SHeiko Carstens do_sigbus(regs);
34036bf9680SMartin Schwidefsky } else
341aa33c8cbSMartin Schwidefsky BUG();
34250d7280dSMartin Schwidefsky break;
34350d7280dSMartin Schwidefsky }
34450d7280dSMartin Schwidefsky }
34550d7280dSMartin Schwidefsky
34650d7280dSMartin Schwidefsky /*
34750d7280dSMartin Schwidefsky * This routine handles page faults. It determines the address,
3481da177e4SLinus Torvalds * and the problem, and then passes it off to one of the appropriate
3491da177e4SLinus Torvalds * routines.
3501da177e4SLinus Torvalds *
3511da177e4SLinus Torvalds * interruption code (int_code):
3521da177e4SLinus Torvalds * 04 Protection -> Write-Protection (suppression)
35350d7280dSMartin Schwidefsky * 10 Segment translation -> Not present (nullification)
3547904aaa8SHeiko Carstens * 11 Page translation -> Not present (nullification)
3551da177e4SLinus Torvalds * 3b Region third trans. -> Not present (nullification)
3561da177e4SLinus Torvalds */
do_exception(struct pt_regs * regs,int access)3571da177e4SLinus Torvalds static inline vm_fault_t do_exception(struct pt_regs *regs, int access)
3581da177e4SLinus Torvalds {
35950a7ca3cSSouptick Joarder struct gmap *gmap;
3601da177e4SLinus Torvalds struct task_struct *tsk;
36124eb3a82SDominik Dingel struct mm_struct *mm;
3621da177e4SLinus Torvalds struct vm_area_struct *vma;
3631da177e4SLinus Torvalds enum fault_type type;
3641da177e4SLinus Torvalds unsigned long address;
3650aaba41bSMartin Schwidefsky unsigned int flags;
3661da177e4SLinus Torvalds vm_fault_t fault;
36733ce6140SHeiko Carstens bool is_write;
36850a7ca3cSSouptick Joarder
369e41ba111SSven Schnelle tsk = current;
37010c1031fSMartin Schwidefsky /*
37139efd4ecSMartin Schwidefsky * The instruction that caused the program check has
37239efd4ecSMartin Schwidefsky * been nullified. Don't signal single step via SIGTRAP.
37339efd4ecSMartin Schwidefsky */
37439efd4ecSMartin Schwidefsky clear_thread_flag(TIF_PER_TRAP);
37539efd4ecSMartin Schwidefsky
37656e62a73SSven Schnelle if (kprobe_page_fault(regs, 14))
37739efd4ecSMartin Schwidefsky return 0;
378b98cca44SAnshuman Khandual
37950d7280dSMartin Schwidefsky mm = tsk->mm;
3801da177e4SLinus Torvalds address = get_fault_address(regs);
3811da177e4SLinus Torvalds is_write = fault_is_write(regs);
382d9c2cf67SHeiko Carstens
383d9c2cf67SHeiko Carstens /*
3841da177e4SLinus Torvalds * Verify that the fault happened in user space, that
3851da177e4SLinus Torvalds * we are not in an interrupt and that there is a
3861da177e4SLinus Torvalds * user context.
3871da177e4SLinus Torvalds */
3881da177e4SLinus Torvalds fault = VM_FAULT_BADCONTEXT;
3891da177e4SLinus Torvalds type = get_fault_type(regs);
39050d7280dSMartin Schwidefsky switch (type) {
3910aaba41bSMartin Schwidefsky case KERNEL_FAULT:
3920aaba41bSMartin Schwidefsky goto out;
3930aaba41bSMartin Schwidefsky case USER_FAULT:
39450d7280dSMartin Schwidefsky case GMAP_FAULT:
3950aaba41bSMartin Schwidefsky if (faulthandler_disabled() || !mm)
3960aaba41bSMartin Schwidefsky goto out;
3970aaba41bSMartin Schwidefsky break;
3980aaba41bSMartin Schwidefsky }
3990aaba41bSMartin Schwidefsky
4000aaba41bSMartin Schwidefsky perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address);
4011da177e4SLinus Torvalds flags = FAULT_FLAG_DEFAULT;
402a8b0ca17SPeter Zijlstra if (user_mode(regs))
403dde16072SPeter Xu flags |= FAULT_FLAG_USER;
404759496baSJohannes Weiner if (is_write)
405759496baSJohannes Weiner access = VM_WRITE;
40641ac42f1SGerald Schaefer if (access == VM_WRITE)
40741ac42f1SGerald Schaefer flags |= FAULT_FLAG_WRITE;
40841ac42f1SGerald Schaefer if (!(flags & FAULT_FLAG_USER))
40933ce6140SHeiko Carstens goto lock_mmap;
410e06f47a1SHeiko Carstens vma = lock_vma_under_rcu(mm, address);
411e06f47a1SHeiko Carstens if (!vma)
412e06f47a1SHeiko Carstens goto lock_mmap;
413e06f47a1SHeiko Carstens if (!(vma->vm_flags & access)) {
414e06f47a1SHeiko Carstens vma_end_read(vma);
415e06f47a1SHeiko Carstens goto lock_mmap;
416e06f47a1SHeiko Carstens }
417e06f47a1SHeiko Carstens fault = handle_mm_fault(vma, address, flags | FAULT_FLAG_VMA_LOCK, regs);
418e06f47a1SHeiko Carstens if (!(fault & (VM_FAULT_RETRY | VM_FAULT_COMPLETED)))
419e06f47a1SHeiko Carstens vma_end_read(vma);
420e06f47a1SHeiko Carstens if (!(fault & VM_FAULT_RETRY)) {
421e06f47a1SHeiko Carstens count_vm_vma_lock_event(VMA_LOCK_SUCCESS);
422e06f47a1SHeiko Carstens if (likely(!(fault & VM_FAULT_ERROR)))
423e06f47a1SHeiko Carstens fault = 0;
424e06f47a1SHeiko Carstens goto out;
425e06f47a1SHeiko Carstens }
426e06f47a1SHeiko Carstens count_vm_vma_lock_event(VMA_LOCK_RETRY);
427e06f47a1SHeiko Carstens /* Quick path to respond to signals */
428e06f47a1SHeiko Carstens if (fault_signal_pending(fault, regs)) {
429e06f47a1SHeiko Carstens fault = VM_FAULT_SIGNAL;
430e06f47a1SHeiko Carstens goto out;
431e06f47a1SHeiko Carstens }
432e06f47a1SHeiko Carstens lock_mmap:
433e06f47a1SHeiko Carstens mmap_read_lock(mm);
434d8ed45c5SMichel Lespinasse
4351da177e4SLinus Torvalds gmap = NULL;
4360aaba41bSMartin Schwidefsky if (IS_ENABLED(CONFIG_PGSTE) && type == GMAP_FAULT) {
4370aaba41bSMartin Schwidefsky gmap = (struct gmap *) S390_lowcore.gmap;
4380aaba41bSMartin Schwidefsky current->thread.gmap_addr = address;
439527e30b4SMartin Schwidefsky current->thread.gmap_write_flag = !!(flags & FAULT_FLAG_WRITE);
4404be130a0SMartin Schwidefsky current->thread.gmap_int_code = regs->int_code & 0xffff;
4414a494439SDavid Hildenbrand address = __gmap_translate(gmap, address);
442527e30b4SMartin Schwidefsky if (address == -EFAULT) {
443e5992f2eSMartin Schwidefsky fault = VM_FAULT_BADMAP;
444e5992f2eSMartin Schwidefsky goto out_up;
445e5992f2eSMartin Schwidefsky }
446e5992f2eSMartin Schwidefsky if (gmap->pfault_enabled)
44724eb3a82SDominik Dingel flags |= FAULT_FLAG_RETRY_NOWAIT;
44824eb3a82SDominik Dingel }
449e5992f2eSMartin Schwidefsky
450e5992f2eSMartin Schwidefsky retry:
451e5992f2eSMartin Schwidefsky fault = VM_FAULT_BADMAP;
45250d7280dSMartin Schwidefsky vma = find_vma(mm, address);
4531da177e4SLinus Torvalds if (!vma)
4541da177e4SLinus Torvalds goto out_up;
45550d7280dSMartin Schwidefsky
456c1821c2eSGerald Schaefer if (unlikely(vma->vm_start > address)) {
45750d7280dSMartin Schwidefsky if (!(vma->vm_flags & VM_GROWSDOWN))
4581da177e4SLinus Torvalds goto out_up;
45950d7280dSMartin Schwidefsky vma = expand_stack(mm, address);
4608d7071afSLinus Torvalds if (!vma)
4618d7071afSLinus Torvalds goto out;
4628d7071afSLinus Torvalds }
46350d7280dSMartin Schwidefsky
46450d7280dSMartin Schwidefsky /*
4651da177e4SLinus Torvalds * Ok, we have a good vm_area for this memory access, so
4661da177e4SLinus Torvalds * we can handle it..
4671da177e4SLinus Torvalds */
4681da177e4SLinus Torvalds fault = VM_FAULT_BADACCESS;
46950d7280dSMartin Schwidefsky if (unlikely(!(vma->vm_flags & access)))
4701ab947deSMartin Schwidefsky goto out_up;
47150d7280dSMartin Schwidefsky
4721da177e4SLinus Torvalds /*
4731da177e4SLinus Torvalds * If for any reason at all we couldn't handle the fault,
4741da177e4SLinus Torvalds * make sure we exit gracefully rather than endlessly redo
4751da177e4SLinus Torvalds * the fault.
4761da177e4SLinus Torvalds */
4771da177e4SLinus Torvalds fault = handle_mm_fault(vma, address, flags, regs);
47835e45f3eSPeter Xu if (fault_signal_pending(fault, regs)) {
4794ef87322SPeter Xu fault = VM_FAULT_SIGNAL;
480f2c76e3bSHeiko Carstens if (flags & FAULT_FLAG_RETRY_NOWAIT)
481306d6c49SClaudio Imbrenda goto out_up;
482306d6c49SClaudio Imbrenda goto out;
483f2c76e3bSHeiko Carstens }
484f2c76e3bSHeiko Carstens
485d9272525SPeter Xu /* The fault is fully completed (including releasing mmap lock) */
486d9272525SPeter Xu if (fault & VM_FAULT_COMPLETED) {
487d9272525SPeter Xu if (gmap) {
488d9272525SPeter Xu mmap_read_lock(mm);
489d9272525SPeter Xu goto out_gmap;
490d9272525SPeter Xu }
491d9272525SPeter Xu fault = 0;
492d9272525SPeter Xu goto out;
493d9272525SPeter Xu }
494d9272525SPeter Xu
495d9272525SPeter Xu if (unlikely(fault & VM_FAULT_ERROR))
49650d7280dSMartin Schwidefsky goto out_up;
49750d7280dSMartin Schwidefsky
49850d7280dSMartin Schwidefsky if (fault & VM_FAULT_RETRY) {
49933ce6140SHeiko Carstens if (IS_ENABLED(CONFIG_PGSTE) && gmap &&
5000aaba41bSMartin Schwidefsky (flags & FAULT_FLAG_RETRY_NOWAIT)) {
5010aaba41bSMartin Schwidefsky /*
50236ef159fSQi Zheng * FAULT_FLAG_RETRY_NOWAIT has been set, mmap_lock has
50336ef159fSQi Zheng * not been released
50436ef159fSQi Zheng */
50536ef159fSQi Zheng current->thread.gmap_pfault = 1;
50624eb3a82SDominik Dingel fault = VM_FAULT_PFAULT;
50724eb3a82SDominik Dingel goto out_up;
50824eb3a82SDominik Dingel }
50924eb3a82SDominik Dingel flags &= ~FAULT_FLAG_RETRY_NOWAIT;
5104064b982SPeter Xu flags |= FAULT_FLAG_TRIED;
51145cac65bSShaohua Li mmap_read_lock(mm);
512d8ed45c5SMichel Lespinasse goto retry;
51333ce6140SHeiko Carstens }
51433ce6140SHeiko Carstens out_gmap:
515d9272525SPeter Xu if (IS_ENABLED(CONFIG_PGSTE) && gmap) {
5160aaba41bSMartin Schwidefsky address = __gmap_link(gmap, current->thread.gmap_addr,
517527e30b4SMartin Schwidefsky address);
518527e30b4SMartin Schwidefsky if (address == -EFAULT) {
519527e30b4SMartin Schwidefsky fault = VM_FAULT_BADMAP;
520527e30b4SMartin Schwidefsky goto out_up;
521527e30b4SMartin Schwidefsky }
522527e30b4SMartin Schwidefsky if (address == -ENOMEM) {
523527e30b4SMartin Schwidefsky fault = VM_FAULT_OOM;
524527e30b4SMartin Schwidefsky goto out_up;
525527e30b4SMartin Schwidefsky }
526527e30b4SMartin Schwidefsky }
527527e30b4SMartin Schwidefsky fault = 0;
52850d7280dSMartin Schwidefsky out_up:
52950d7280dSMartin Schwidefsky mmap_read_unlock(mm);
530d8ed45c5SMichel Lespinasse out:
53150d7280dSMartin Schwidefsky return fault;
53250d7280dSMartin Schwidefsky }
5331da177e4SLinus Torvalds
do_protection_exception(struct pt_regs * regs)5341da177e4SLinus Torvalds void do_protection_exception(struct pt_regs *regs)
5357a5388deSHeiko Carstens {
5361da177e4SLinus Torvalds unsigned long trans_exc_code;
537aa33c8cbSMartin Schwidefsky int access;
53850a7ca3cSSouptick Joarder vm_fault_t fault;
53950a7ca3cSSouptick Joarder
54061365e13SMartin Schwidefsky trans_exc_code = regs->int_parm_long;
541aa33c8cbSMartin Schwidefsky /*
542f752ac4dSMartin Schwidefsky * Protection exceptions are suppressing, decrement psw address.
543f752ac4dSMartin Schwidefsky * The exception to this rule are aborted transactions, for these
544f752ac4dSMartin Schwidefsky * the PSW already points to the correct location.
545f752ac4dSMartin Schwidefsky */
546f752ac4dSMartin Schwidefsky if (!(regs->int_code & 0x200))
547f752ac4dSMartin Schwidefsky regs->psw.addr = __rewind_psw(regs->psw, regs->int_code >> 16);
548aa33c8cbSMartin Schwidefsky /*
54910c1031fSMartin Schwidefsky * Check for low-address protection. This needs to be treated
55010c1031fSMartin Schwidefsky * as a special case because the translation exception code
55110c1031fSMartin Schwidefsky * field is not guaranteed to contain valid data in this case.
55210c1031fSMartin Schwidefsky */
55310c1031fSMartin Schwidefsky if (unlikely(!(trans_exc_code & 4))) {
55461365e13SMartin Schwidefsky do_low_address(regs);
555aa33c8cbSMartin Schwidefsky return;
55610c1031fSMartin Schwidefsky }
55710c1031fSMartin Schwidefsky if (unlikely(MACHINE_HAS_NX && (trans_exc_code & 0x80))) {
55857d7f939SMartin Schwidefsky regs->int_parm_long = (trans_exc_code & ~PAGE_MASK) |
55957d7f939SMartin Schwidefsky (regs->psw.addr & PAGE_MASK);
56057d7f939SMartin Schwidefsky access = VM_EXEC;
56157d7f939SMartin Schwidefsky fault = VM_FAULT_BADACCESS;
56257d7f939SMartin Schwidefsky } else {
56357d7f939SMartin Schwidefsky access = VM_WRITE;
56457d7f939SMartin Schwidefsky fault = do_exception(regs, access);
56557d7f939SMartin Schwidefsky }
56657d7f939SMartin Schwidefsky if (unlikely(fault))
56750d7280dSMartin Schwidefsky do_fault_error(regs, fault);
568bf2ce385SHeiko Carstens }
5691da177e4SLinus Torvalds NOKPROBE_SYMBOL(do_protection_exception);
5707a5388deSHeiko Carstens
do_dat_exception(struct pt_regs * regs)5711da177e4SLinus Torvalds void do_dat_exception(struct pt_regs *regs)
5727a5388deSHeiko Carstens {
5731da177e4SLinus Torvalds int access;
57450a7ca3cSSouptick Joarder vm_fault_t fault;
57550a7ca3cSSouptick Joarder
57650d7280dSMartin Schwidefsky access = VM_ACCESS_FLAGS;
5776cb4d9a2SAnshuman Khandual fault = do_exception(regs, access);
578aa33c8cbSMartin Schwidefsky if (unlikely(fault))
57950d7280dSMartin Schwidefsky do_fault_error(regs, fault);
580bf2ce385SHeiko Carstens }
5811da177e4SLinus Torvalds NOKPROBE_SYMBOL(do_dat_exception);
5827a5388deSHeiko Carstens
5831da177e4SLinus Torvalds #if IS_ENABLED(CONFIG_PGSTE)
5841da177e4SLinus Torvalds
do_secure_storage_access(struct pt_regs * regs)5851da177e4SLinus Torvalds void do_secure_storage_access(struct pt_regs *regs)
5861da177e4SLinus Torvalds {
5871da177e4SLinus Torvalds unsigned long addr = regs->int_parm_long & __FAIL_ADDR_MASK;
588fb0a9d7eSHeiko Carstens struct vm_area_struct *vma;
5891da177e4SLinus Torvalds struct mm_struct *mm;
5901da177e4SLinus Torvalds struct page *page;
5911da177e4SLinus Torvalds struct gmap *gmap;
5921da177e4SLinus Torvalds int rc;
5931da177e4SLinus Torvalds
5941da177e4SLinus Torvalds /*
5951da177e4SLinus Torvalds * bit 61 tells us if the address is valid, if it's not we
5961da177e4SLinus Torvalds * have a major problem and should stop the kernel or send a
5971da177e4SLinus Torvalds * SIGSEGV to the process. Unfortunately bit 61 is not
5987dd8fe1fSHeiko Carstens * reliable without the misc UV feature so we need to check
5997dd8fe1fSHeiko Carstens * for that as well.
6007dd8fe1fSHeiko Carstens */
6017dd8fe1fSHeiko Carstens if (uv_has_feature(BIT_UV_FEAT_MISC) &&
6027dd8fe1fSHeiko Carstens !test_bit_inv(61, ®s->int_parm_long)) {
6037dd8fe1fSHeiko Carstens /*
6047dd8fe1fSHeiko Carstens * When this happens, userspace did something that it
6057dd8fe1fSHeiko Carstens * was not supposed to do, e.g. branching into secure
6067dd8fe1fSHeiko Carstens * memory. Trigger a segmentation fault.
6077dd8fe1fSHeiko Carstens */
6081da177e4SLinus Torvalds if (user_mode(regs)) {
60900e9e664SMartin Schwidefsky send_sig(SIGSEGV, current, 0);
6107dd8fe1fSHeiko Carstens return;
6117dd8fe1fSHeiko Carstens }
6127dd8fe1fSHeiko Carstens
6137dd8fe1fSHeiko Carstens /*
614e22cf8caSChristian Borntraeger * The kernel should never run into this case and we
6157dd8fe1fSHeiko Carstens * have no way out of this situation.
6167dd8fe1fSHeiko Carstens */
61700e9e664SMartin Schwidefsky panic("Unexpected PGM 0x3d with TEID bit 61=0");
61800e9e664SMartin Schwidefsky }
61900e9e664SMartin Schwidefsky
62000e9e664SMartin Schwidefsky switch (get_fault_type(regs)) {
62100e9e664SMartin Schwidefsky case GMAP_FAULT:
6221da177e4SLinus Torvalds mm = current->mm;
6231da177e4SLinus Torvalds gmap = (struct gmap *)S390_lowcore.gmap;
624f32269a0SCarsten Otte mmap_read_lock(mm);
6251da177e4SLinus Torvalds addr = __gmap_translate(gmap, addr);
6261ec2772eSMartin Schwidefsky mmap_read_unlock(mm);
62794c12cc7SMartin Schwidefsky if (IS_ERR_VALUE(addr)) {
6281da177e4SLinus Torvalds do_fault_error(regs, VM_FAULT_BADMAP);
6291da177e4SLinus Torvalds break;
6301da177e4SLinus Torvalds }
6311da177e4SLinus Torvalds fallthrough;
63294c12cc7SMartin Schwidefsky case USER_FAULT:
63300e9e664SMartin Schwidefsky mm = current->mm;
63400e9e664SMartin Schwidefsky mmap_read_lock(mm);
6351da177e4SLinus Torvalds vma = find_vma(mm, addr);
6361da177e4SLinus Torvalds if (!vma) {
6371da177e4SLinus Torvalds mmap_read_unlock(mm);
63800e9e664SMartin Schwidefsky do_fault_error(regs, VM_FAULT_BADMAP);
6397dd8fe1fSHeiko Carstens break;
6407dd8fe1fSHeiko Carstens }
6417dd8fe1fSHeiko Carstens page = follow_page(vma, addr, FOLL_WRITE | FOLL_GET);
6427dd8fe1fSHeiko Carstens if (IS_ERR_OR_NULL(page)) {
6437dd8fe1fSHeiko Carstens mmap_read_unlock(mm);
6441da177e4SLinus Torvalds break;
64500e9e664SMartin Schwidefsky }
64600e9e664SMartin Schwidefsky if (arch_make_page_accessible(page))
64700e9e664SMartin Schwidefsky send_sig(SIGSEGV, current, 0);
648f32269a0SCarsten Otte put_page(page);
6491da177e4SLinus Torvalds mmap_read_unlock(mm);
6501ec2772eSMartin Schwidefsky break;
65194c12cc7SMartin Schwidefsky case KERNEL_FAULT:
6521da177e4SLinus Torvalds page = phys_to_page(addr);
6536c22c986SHeiko Carstens if (unlikely(!try_get_page(page)))
65494c12cc7SMartin Schwidefsky break;
65500e9e664SMartin Schwidefsky rc = arch_make_page_accessible(page);
6561da177e4SLinus Torvalds put_page(page);
6571da177e4SLinus Torvalds if (rc)
658f2db2e6cSHeiko Carstens BUG();
659f2db2e6cSHeiko Carstens break;
660f2db2e6cSHeiko Carstens default:
6610227f7c4SPeter Zijlstra do_fault_error(regs, VM_FAULT_BADMAP);
6620227f7c4SPeter Zijlstra WARN_ON_ONCE(1);
6630227f7c4SPeter Zijlstra }
6640227f7c4SPeter Zijlstra }
6650227f7c4SPeter Zijlstra NOKPROBE_SYMBOL(do_secure_storage_access);
6660227f7c4SPeter Zijlstra
do_non_secure_storage_access(struct pt_regs * regs)6670227f7c4SPeter Zijlstra void do_non_secure_storage_access(struct pt_regs *regs)
6680227f7c4SPeter Zijlstra {
6690227f7c4SPeter Zijlstra unsigned long gaddr = regs->int_parm_long & __FAIL_ADDR_MASK;
6700227f7c4SPeter Zijlstra struct gmap *gmap = (struct gmap *)S390_lowcore.gmap;
6710227f7c4SPeter Zijlstra
6720227f7c4SPeter Zijlstra if (get_fault_type(regs) != GMAP_FAULT) {
6730227f7c4SPeter Zijlstra do_fault_error(regs, VM_FAULT_BADMAP);
6740227f7c4SPeter Zijlstra WARN_ON_ONCE(1);
6750227f7c4SPeter Zijlstra return;
6760227f7c4SPeter Zijlstra }
6770227f7c4SPeter Zijlstra
6780227f7c4SPeter Zijlstra if (gmap_convert_to_secure(gmap, gaddr) == -EINVAL)
6790227f7c4SPeter Zijlstra send_sig(SIGSEGV, current, 0);
6800227f7c4SPeter Zijlstra }
6810227f7c4SPeter Zijlstra NOKPROBE_SYMBOL(do_non_secure_storage_access);
6820227f7c4SPeter Zijlstra
do_secure_storage_violation(struct pt_regs * regs)6830227f7c4SPeter Zijlstra void do_secure_storage_violation(struct pt_regs *regs)
684fde15c3aSHeiko Carstens {
685f6649a7eSMartin Schwidefsky unsigned long gaddr = regs->int_parm_long & __FAIL_ADDR_MASK;
6861da177e4SLinus Torvalds struct gmap *gmap = (struct gmap *)S390_lowcore.gmap;
6871da177e4SLinus Torvalds
6881da177e4SLinus Torvalds /*
689f2db2e6cSHeiko Carstens * If the VM has been rebooted, its address space might still contain
6901da177e4SLinus Torvalds * secure pages from the previous boot.
6911da177e4SLinus Torvalds * Clear the page so it can be reused.
6920227f7c4SPeter Zijlstra */
6930227f7c4SPeter Zijlstra if (!gmap_destroy_page(gmap, gaddr))
6940227f7c4SPeter Zijlstra return;
6951da177e4SLinus Torvalds /*
696fde15c3aSHeiko Carstens * Either KVM messed up the secure guest mapping or the same
6971da177e4SLinus Torvalds * page is mapped into multiple secure guests.
6981da177e4SLinus Torvalds *
699420f42ecSHeiko Carstens * This exception is only triggered when a guest 2 is running
700f2db2e6cSHeiko Carstens * and can therefore never occur in kernel context.
701544e8dd7SHendrik Brueckner */
702f2db2e6cSHeiko Carstens printk_ratelimited(KERN_WARNING
703f2db2e6cSHeiko Carstens "Secure storage violation in task: %s, pid %d\n",
704f2db2e6cSHeiko Carstens current->comm, current->pid);
705f2db2e6cSHeiko Carstens send_sig(SIGSEGV, current, 0);
706f2db2e6cSHeiko Carstens }
707f2db2e6cSHeiko Carstens
708f2db2e6cSHeiko Carstens #endif /* CONFIG_PGSTE */
709f2db2e6cSHeiko Carstens