119f6d2a6SOded Gabbay /* 219f6d2a6SOded Gabbay * Copyright 2014 Advanced Micro Devices, Inc. 319f6d2a6SOded Gabbay * 419f6d2a6SOded Gabbay * Permission is hereby granted, free of charge, to any person obtaining a 519f6d2a6SOded Gabbay * copy of this software and associated documentation files (the "Software"), 619f6d2a6SOded Gabbay * to deal in the Software without restriction, including without limitation 719f6d2a6SOded Gabbay * the rights to use, copy, modify, merge, publish, distribute, sublicense, 819f6d2a6SOded Gabbay * and/or sell copies of the Software, and to permit persons to whom the 919f6d2a6SOded Gabbay * Software is furnished to do so, subject to the following conditions: 1019f6d2a6SOded Gabbay * 1119f6d2a6SOded Gabbay * The above copyright notice and this permission notice shall be included in 1219f6d2a6SOded Gabbay * all copies or substantial portions of the Software. 1319f6d2a6SOded Gabbay * 1419f6d2a6SOded Gabbay * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 1519f6d2a6SOded Gabbay * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 1619f6d2a6SOded Gabbay * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 1719f6d2a6SOded Gabbay * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 1819f6d2a6SOded Gabbay * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 1919f6d2a6SOded Gabbay * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 2019f6d2a6SOded Gabbay * OTHER DEALINGS IN THE SOFTWARE. 2119f6d2a6SOded Gabbay * 2219f6d2a6SOded Gabbay */ 2319f6d2a6SOded Gabbay 2419f6d2a6SOded Gabbay #include <linux/device.h> 2519f6d2a6SOded Gabbay #include <linux/export.h> 2619f6d2a6SOded Gabbay #include <linux/err.h> 2719f6d2a6SOded Gabbay #include <linux/fs.h> 2819f6d2a6SOded Gabbay #include <linux/sched.h> 2919f6d2a6SOded Gabbay #include <linux/slab.h> 3019f6d2a6SOded Gabbay #include <linux/uaccess.h> 3119f6d2a6SOded Gabbay #include <linux/compat.h> 3219f6d2a6SOded Gabbay #include <uapi/linux/kfd_ioctl.h> 3319f6d2a6SOded Gabbay #include <linux/time.h> 3419f6d2a6SOded Gabbay #include "kfd_priv.h" 3519f6d2a6SOded Gabbay #include <linux/mm.h> 362497ee72SChristoph Hellwig #include <linux/mman.h> 3719f6d2a6SOded Gabbay #include <asm/processor.h> 3819f6d2a6SOded Gabbay 3919f6d2a6SOded Gabbay /* 4019f6d2a6SOded Gabbay * The primary memory I/O features being added for revisions of gfxip 4119f6d2a6SOded Gabbay * beyond 7.0 (Kaveri) are: 4219f6d2a6SOded Gabbay * 4319f6d2a6SOded Gabbay * Access to ATC/IOMMU mapped memory w/ associated extension of VA to 48b 4419f6d2a6SOded Gabbay * 4519f6d2a6SOded Gabbay * “Flat” shader memory access – These are new shader vector memory 4619f6d2a6SOded Gabbay * operations that do not reference a T#/V# so a “pointer” is what is 4719f6d2a6SOded Gabbay * sourced from the vector gprs for direct access to memory. 4819f6d2a6SOded Gabbay * This pointer space has the Shared(LDS) and Private(Scratch) memory 4919f6d2a6SOded Gabbay * mapped into this pointer space as apertures. 5019f6d2a6SOded Gabbay * The hardware then determines how to direct the memory request 5119f6d2a6SOded Gabbay * based on what apertures the request falls in. 5219f6d2a6SOded Gabbay * 5319f6d2a6SOded Gabbay * Unaligned support and alignment check 5419f6d2a6SOded Gabbay * 5519f6d2a6SOded Gabbay * 5619f6d2a6SOded Gabbay * System Unified Address - SUA 5719f6d2a6SOded Gabbay * 5819f6d2a6SOded Gabbay * The standard usage for GPU virtual addresses are that they are mapped by 5919f6d2a6SOded Gabbay * a set of page tables we call GPUVM and these page tables are managed by 6019f6d2a6SOded Gabbay * a combination of vidMM/driver software components. The current virtual 6119f6d2a6SOded Gabbay * address (VA) range for GPUVM is 40b. 6219f6d2a6SOded Gabbay * 6319f6d2a6SOded Gabbay * As of gfxip7.1 and beyond we’re adding the ability for compute memory 6419f6d2a6SOded Gabbay * clients (CP/RLC, DMA, SHADER(ifetch, scalar, and vector ops)) to access 6519f6d2a6SOded Gabbay * the same page tables used by host x86 processors and that are managed by 6619f6d2a6SOded Gabbay * the operating system. This is via a technique and hardware called ATC/IOMMU. 6719f6d2a6SOded Gabbay * The GPU has the capability of accessing both the GPUVM and ATC address 6819f6d2a6SOded Gabbay * spaces for a given VMID (process) simultaneously and we call this feature 6919f6d2a6SOded Gabbay * system unified address (SUA). 7019f6d2a6SOded Gabbay * 7119f6d2a6SOded Gabbay * There are three fundamental address modes of operation for a given VMID 7219f6d2a6SOded Gabbay * (process) on the GPU: 7319f6d2a6SOded Gabbay * 7419f6d2a6SOded Gabbay * HSA64 – 64b pointers and the default address space is ATC 7519f6d2a6SOded Gabbay * HSA32 – 32b pointers and the default address space is ATC 7619f6d2a6SOded Gabbay * GPUVM – 64b pointers and the default address space is GPUVM (driver 7719f6d2a6SOded Gabbay * model mode) 7819f6d2a6SOded Gabbay * 7919f6d2a6SOded Gabbay * 8019f6d2a6SOded Gabbay * HSA64 - ATC/IOMMU 64b 8119f6d2a6SOded Gabbay * 8219f6d2a6SOded Gabbay * A 64b pointer in the AMD64/IA64 CPU architecture is not fully utilized 8319f6d2a6SOded Gabbay * by the CPU so an AMD CPU can only access the high area 8419f6d2a6SOded Gabbay * (VA[63:47] == 0x1FFFF) and low area (VA[63:47 == 0) of the address space 8519f6d2a6SOded Gabbay * so the actual VA carried to translation is 48b. There is a “hole” in 8619f6d2a6SOded Gabbay * the middle of the 64b VA space. 8719f6d2a6SOded Gabbay * 8819f6d2a6SOded Gabbay * The GPU not only has access to all of the CPU accessible address space via 8919f6d2a6SOded Gabbay * ATC/IOMMU, but it also has access to the GPUVM address space. The “system 9019f6d2a6SOded Gabbay * unified address” feature (SUA) is the mapping of GPUVM and ATC address 9119f6d2a6SOded Gabbay * spaces into a unified pointer space. The method we take for 64b mode is 9219f6d2a6SOded Gabbay * to map the full 40b GPUVM address space into the hole of the 64b address 9319f6d2a6SOded Gabbay * space. 9419f6d2a6SOded Gabbay 9519f6d2a6SOded Gabbay * The GPUVM_Base/GPUVM_Limit defines the aperture in the 64b space where we 9619f6d2a6SOded Gabbay * direct requests to be translated via GPUVM page tables instead of the 9719f6d2a6SOded Gabbay * IOMMU path. 9819f6d2a6SOded Gabbay * 9919f6d2a6SOded Gabbay * 10019f6d2a6SOded Gabbay * 64b to 49b Address conversion 10119f6d2a6SOded Gabbay * 10219f6d2a6SOded Gabbay * Note that there are still significant portions of unused regions (holes) 10319f6d2a6SOded Gabbay * in the 64b address space even for the GPU. There are several places in 10419f6d2a6SOded Gabbay * the pipeline (sw and hw), we wish to compress the 64b virtual address 10519f6d2a6SOded Gabbay * to a 49b address. This 49b address is constituted of an “ATC” bit 10619f6d2a6SOded Gabbay * plus a 48b virtual address. This 49b address is what is passed to the 10719f6d2a6SOded Gabbay * translation hardware. ATC==0 means the 48b address is a GPUVM address 10819f6d2a6SOded Gabbay * (max of 2^40 – 1) intended to be translated via GPUVM page tables. 10919f6d2a6SOded Gabbay * ATC==1 means the 48b address is intended to be translated via IOMMU 11019f6d2a6SOded Gabbay * page tables. 11119f6d2a6SOded Gabbay * 11219f6d2a6SOded Gabbay * A 64b pointer is compared to the apertures that are defined (Base/Limit), in 11319f6d2a6SOded Gabbay * this case the GPUVM aperture (red) is defined and if a pointer falls in this 11419f6d2a6SOded Gabbay * aperture, we subtract the GPUVM_Base address and set the ATC bit to zero 11519f6d2a6SOded Gabbay * as part of the 64b to 49b conversion. 11619f6d2a6SOded Gabbay * 11719f6d2a6SOded Gabbay * Where this 64b to 49b conversion is done is a function of the usage. 11819f6d2a6SOded Gabbay * Most GPU memory access is via memory objects where the driver builds 11919f6d2a6SOded Gabbay * a descriptor which consists of a base address and a memory access by 12019f6d2a6SOded Gabbay * the GPU usually consists of some kind of an offset or Cartesian coordinate 12119f6d2a6SOded Gabbay * that references this memory descriptor. This is the case for shader 12219f6d2a6SOded Gabbay * instructions that reference the T# or V# constants, or for specified 12319f6d2a6SOded Gabbay * locations of assets (ex. the shader program location). In these cases 12419f6d2a6SOded Gabbay * the driver is what handles the 64b to 49b conversion and the base 12519f6d2a6SOded Gabbay * address in the descriptor (ex. V# or T# or shader program location) 12619f6d2a6SOded Gabbay * is defined as a 48b address w/ an ATC bit. For this usage a given 12719f6d2a6SOded Gabbay * memory object cannot straddle multiple apertures in the 64b address 12819f6d2a6SOded Gabbay * space. For example a shader program cannot jump in/out between ATC 12919f6d2a6SOded Gabbay * and GPUVM space. 13019f6d2a6SOded Gabbay * 13119f6d2a6SOded Gabbay * In some cases we wish to pass a 64b pointer to the GPU hardware and 13219f6d2a6SOded Gabbay * the GPU hw does the 64b to 49b conversion before passing memory 13319f6d2a6SOded Gabbay * requests to the cache/memory system. This is the case for the 13419f6d2a6SOded Gabbay * S_LOAD and FLAT_* shader memory instructions where we have 64b pointers 13519f6d2a6SOded Gabbay * in scalar and vector GPRs respectively. 13619f6d2a6SOded Gabbay * 13719f6d2a6SOded Gabbay * In all cases (no matter where the 64b -> 49b conversion is done), the gfxip 13819f6d2a6SOded Gabbay * hardware sends a 48b address along w/ an ATC bit, to the memory controller 13919f6d2a6SOded Gabbay * on the memory request interfaces. 14019f6d2a6SOded Gabbay * 14119f6d2a6SOded Gabbay * <client>_MC_rdreq_atc // read request ATC bit 14219f6d2a6SOded Gabbay * 14319f6d2a6SOded Gabbay * 0 : <client>_MC_rdreq_addr is a GPUVM VA 14419f6d2a6SOded Gabbay * 14519f6d2a6SOded Gabbay * 1 : <client>_MC_rdreq_addr is a ATC VA 14619f6d2a6SOded Gabbay * 14719f6d2a6SOded Gabbay * 14819f6d2a6SOded Gabbay * “Spare” aperture (APE1) 14919f6d2a6SOded Gabbay * 15019f6d2a6SOded Gabbay * We use the GPUVM aperture to differentiate ATC vs. GPUVM, but we also use 15119f6d2a6SOded Gabbay * apertures to set the Mtype field for S_LOAD/FLAT_* ops which is input to the 15219f6d2a6SOded Gabbay * config tables for setting cache policies. The “spare” (APE1) aperture is 15319f6d2a6SOded Gabbay * motivated by getting a different Mtype from the default. 15419f6d2a6SOded Gabbay * The default aperture isn’t an actual base/limit aperture; it is just the 15519f6d2a6SOded Gabbay * address space that doesn’t hit any defined base/limit apertures. 15619f6d2a6SOded Gabbay * The following diagram is a complete picture of the gfxip7.x SUA apertures. 15719f6d2a6SOded Gabbay * The APE1 can be placed either below or above 15819f6d2a6SOded Gabbay * the hole (cannot be in the hole). 15919f6d2a6SOded Gabbay * 16019f6d2a6SOded Gabbay * 16119f6d2a6SOded Gabbay * General Aperture definitions and rules 16219f6d2a6SOded Gabbay * 16319f6d2a6SOded Gabbay * An aperture register definition consists of a Base, Limit, Mtype, and 16419f6d2a6SOded Gabbay * usually an ATC bit indicating which translation tables that aperture uses. 16519f6d2a6SOded Gabbay * In all cases (for SUA and DUA apertures discussed later), aperture base 16619f6d2a6SOded Gabbay * and limit definitions are 64KB aligned. 16719f6d2a6SOded Gabbay * 16819f6d2a6SOded Gabbay * <ape>_Base[63:0] = { <ape>_Base_register[63:16], 0x0000 } 16919f6d2a6SOded Gabbay * 17019f6d2a6SOded Gabbay * <ape>_Limit[63:0] = { <ape>_Limit_register[63:16], 0xFFFF } 17119f6d2a6SOded Gabbay * 17219f6d2a6SOded Gabbay * The base and limit are considered inclusive to an aperture so being 17319f6d2a6SOded Gabbay * inside an aperture means (address >= Base) AND (address <= Limit). 17419f6d2a6SOded Gabbay * 17519f6d2a6SOded Gabbay * In no case is a payload that straddles multiple apertures expected to work. 17619f6d2a6SOded Gabbay * For example a load_dword_x4 that starts in one aperture and ends in another, 17719f6d2a6SOded Gabbay * does not work. For the vector FLAT_* ops we have detection capability in 17819f6d2a6SOded Gabbay * the shader for reporting a “memory violation” back to the 17919f6d2a6SOded Gabbay * SQ block for use in traps. 18019f6d2a6SOded Gabbay * A memory violation results when an op falls into the hole, 18119f6d2a6SOded Gabbay * or a payload straddles multiple apertures. The S_LOAD instruction 18219f6d2a6SOded Gabbay * does not have this detection. 18319f6d2a6SOded Gabbay * 18419f6d2a6SOded Gabbay * Apertures cannot overlap. 18519f6d2a6SOded Gabbay * 18619f6d2a6SOded Gabbay * 18719f6d2a6SOded Gabbay * 18819f6d2a6SOded Gabbay * HSA32 - ATC/IOMMU 32b 18919f6d2a6SOded Gabbay * 19019f6d2a6SOded Gabbay * For HSA32 mode, the pointers are interpreted as 32 bits and use a single GPR 19119f6d2a6SOded Gabbay * instead of two for the S_LOAD and FLAT_* ops. The entire GPUVM space of 40b 19219f6d2a6SOded Gabbay * will not fit so there is only partial visibility to the GPUVM 19319f6d2a6SOded Gabbay * space (defined by the aperture) for S_LOAD and FLAT_* ops. 19419f6d2a6SOded Gabbay * There is no spare (APE1) aperture for HSA32 mode. 19519f6d2a6SOded Gabbay * 19619f6d2a6SOded Gabbay * 19719f6d2a6SOded Gabbay * GPUVM 64b mode (driver model) 19819f6d2a6SOded Gabbay * 19919f6d2a6SOded Gabbay * This mode is related to HSA64 in that the difference really is that 20019f6d2a6SOded Gabbay * the default aperture is GPUVM (ATC==0) and not ATC space. 20119f6d2a6SOded Gabbay * We have gfxip7.x hardware that has FLAT_* and S_LOAD support for 20219f6d2a6SOded Gabbay * SUA GPUVM mode, but does not support HSA32/HSA64. 20319f6d2a6SOded Gabbay * 20419f6d2a6SOded Gabbay * 20519f6d2a6SOded Gabbay * Device Unified Address - DUA 20619f6d2a6SOded Gabbay * 20719f6d2a6SOded Gabbay * Device unified address (DUA) is the name of the feature that maps the 20819f6d2a6SOded Gabbay * Shared(LDS) memory and Private(Scratch) memory into the overall address 20919f6d2a6SOded Gabbay * space for use by the new FLAT_* vector memory ops. The Shared and 21019f6d2a6SOded Gabbay * Private memories are mapped as apertures into the address space, 21119f6d2a6SOded Gabbay * and the hardware detects when a FLAT_* memory request is to be redirected 21219f6d2a6SOded Gabbay * to the LDS or Scratch memory when it falls into one of these apertures. 21319f6d2a6SOded Gabbay * Like the SUA apertures, the Shared/Private apertures are 64KB aligned and 21419f6d2a6SOded Gabbay * the base/limit is “in” the aperture. For both HSA64 and GPUVM SUA modes, 21519f6d2a6SOded Gabbay * the Shared/Private apertures are always placed in a limited selection of 21619f6d2a6SOded Gabbay * options in the hole of the 64b address space. For HSA32 mode, the 21719f6d2a6SOded Gabbay * Shared/Private apertures can be placed anywhere in the 32b space 21819f6d2a6SOded Gabbay * except at 0. 21919f6d2a6SOded Gabbay * 22019f6d2a6SOded Gabbay * 22119f6d2a6SOded Gabbay * HSA64 Apertures for FLAT_* vector ops 22219f6d2a6SOded Gabbay * 22319f6d2a6SOded Gabbay * For HSA64 SUA mode, the Shared and Private apertures are always placed 22419f6d2a6SOded Gabbay * in the hole w/ a limited selection of possible locations. The requests 22519f6d2a6SOded Gabbay * that fall in the private aperture are expanded as a function of the 22619f6d2a6SOded Gabbay * work-item id (tid) and redirected to the location of the 22719f6d2a6SOded Gabbay * “hidden private memory”. The hidden private can be placed in either GPUVM 22819f6d2a6SOded Gabbay * or ATC space. The addresses that fall in the shared aperture are 22919f6d2a6SOded Gabbay * re-directed to the on-chip LDS memory hardware. 23019f6d2a6SOded Gabbay * 23119f6d2a6SOded Gabbay * 23219f6d2a6SOded Gabbay * HSA32 Apertures for FLAT_* vector ops 23319f6d2a6SOded Gabbay * 23419f6d2a6SOded Gabbay * In HSA32 mode, the Private and Shared apertures can be placed anywhere 23519f6d2a6SOded Gabbay * in the 32b space except at 0 (Private or Shared Base at zero disables 23619f6d2a6SOded Gabbay * the apertures). If the base address of the apertures are non-zero 23719f6d2a6SOded Gabbay * (ie apertures exists), the size is always 64KB. 23819f6d2a6SOded Gabbay * 23919f6d2a6SOded Gabbay * 24019f6d2a6SOded Gabbay * GPUVM Apertures for FLAT_* vector ops 24119f6d2a6SOded Gabbay * 24219f6d2a6SOded Gabbay * In GPUVM mode, the Shared/Private apertures are specified identically 24319f6d2a6SOded Gabbay * to HSA64 mode where they are always in the hole at a limited selection 24419f6d2a6SOded Gabbay * of locations. 24519f6d2a6SOded Gabbay * 24619f6d2a6SOded Gabbay * 24719f6d2a6SOded Gabbay * Aperture Definitions for SUA and DUA 24819f6d2a6SOded Gabbay * 24919f6d2a6SOded Gabbay * The interpretation of the aperture register definitions for a given 25019f6d2a6SOded Gabbay * VMID is a function of the “SUA Mode” which is one of HSA64, HSA32, or 25119f6d2a6SOded Gabbay * GPUVM64 discussed in previous sections. The mode is first decoded, and 25219f6d2a6SOded Gabbay * then the remaining register decode is a function of the mode. 25319f6d2a6SOded Gabbay * 25419f6d2a6SOded Gabbay * 25519f6d2a6SOded Gabbay * SUA Mode Decode 25619f6d2a6SOded Gabbay * 25719f6d2a6SOded Gabbay * For the S_LOAD and FLAT_* shader operations, the SUA mode is decoded from 25819f6d2a6SOded Gabbay * the COMPUTE_DISPATCH_INITIATOR:DATA_ATC bit and 25919f6d2a6SOded Gabbay * the SH_MEM_CONFIG:PTR32 bits. 26019f6d2a6SOded Gabbay * 26119f6d2a6SOded Gabbay * COMPUTE_DISPATCH_INITIATOR:DATA_ATC SH_MEM_CONFIG:PTR32 Mode 26219f6d2a6SOded Gabbay * 26319f6d2a6SOded Gabbay * 1 0 HSA64 26419f6d2a6SOded Gabbay * 26519f6d2a6SOded Gabbay * 1 1 HSA32 26619f6d2a6SOded Gabbay * 26719f6d2a6SOded Gabbay * 0 X GPUVM64 26819f6d2a6SOded Gabbay * 26919f6d2a6SOded Gabbay * In general the hardware will ignore the PTR32 bit and treat 27019f6d2a6SOded Gabbay * as “0” whenever DATA_ATC = “0”, but sw should set PTR32=0 27119f6d2a6SOded Gabbay * when DATA_ATC=0. 27219f6d2a6SOded Gabbay * 27319f6d2a6SOded Gabbay * The DATA_ATC bit is only set for compute dispatches. 27419f6d2a6SOded Gabbay * All “Draw” dispatches are hardcoded to GPUVM64 mode 27519f6d2a6SOded Gabbay * for FLAT_* / S_LOAD operations. 27619f6d2a6SOded Gabbay */ 27719f6d2a6SOded Gabbay 27870a31d16SFelix Kuehling #define MAKE_GPUVM_APP_BASE_VI(gpu_num) \ 279585dbf38SOded Gabbay (((uint64_t)(gpu_num) << 61) + 0x1000000000000L) 28019f6d2a6SOded Gabbay 281d01994c2SFelix Kuehling #define MAKE_GPUVM_APP_LIMIT(base, size) \ 282d01994c2SFelix Kuehling (((uint64_t)(base) & 0xFFFFFF0000000000UL) + (size) - 1) 28319f6d2a6SOded Gabbay 28470a31d16SFelix Kuehling #define MAKE_SCRATCH_APP_BASE_VI() \ 285c7bcbfa4SFelix Kuehling (((uint64_t)(0x1UL) << 61) + 0x100000000L) 28619f6d2a6SOded Gabbay 28719f6d2a6SOded Gabbay #define MAKE_SCRATCH_APP_LIMIT(base) \ 288585dbf38SOded Gabbay (((uint64_t)base & 0xFFFFFFFF00000000UL) | 0xFFFFFFFF) 28919f6d2a6SOded Gabbay 29070a31d16SFelix Kuehling #define MAKE_LDS_APP_BASE_VI() \ 291c7bcbfa4SFelix Kuehling (((uint64_t)(0x1UL) << 61) + 0x0) 29219f6d2a6SOded Gabbay #define MAKE_LDS_APP_LIMIT(base) \ 293585dbf38SOded Gabbay (((uint64_t)(base) & 0xFFFFFFFF00000000UL) | 0xFFFFFFFF) 29419f6d2a6SOded Gabbay 29570a31d16SFelix Kuehling /* On GFXv9 the LDS and scratch apertures are programmed independently 29670a31d16SFelix Kuehling * using the high 16 bits of the 64-bit virtual address. They must be 29770a31d16SFelix Kuehling * in the hole, which will be the case as long as the high 16 bits are 29870a31d16SFelix Kuehling * not 0. 29970a31d16SFelix Kuehling * 30070a31d16SFelix Kuehling * The aperture sizes are still 4GB implicitly. 30170a31d16SFelix Kuehling * 30270a31d16SFelix Kuehling * A GPUVM aperture is not applicable on GFXv9. 30370a31d16SFelix Kuehling */ 30470a31d16SFelix Kuehling #define MAKE_LDS_APP_BASE_V9() ((uint64_t)(0x1UL) << 48) 30570a31d16SFelix Kuehling #define MAKE_SCRATCH_APP_BASE_V9() ((uint64_t)(0x2UL) << 48) 30670a31d16SFelix Kuehling 307d01994c2SFelix Kuehling /* User mode manages most of the SVM aperture address space. The low 308d01994c2SFelix Kuehling * 16MB are reserved for kernel use (CWSR trap handler and kernel IB 309d01994c2SFelix Kuehling * for now). 310d01994c2SFelix Kuehling */ 311d01994c2SFelix Kuehling #define SVM_USER_BASE 0x1000000ull 312d01994c2SFelix Kuehling #define SVM_CWSR_BASE (SVM_USER_BASE - KFD_CWSR_TBA_TMA_SIZE) 313d01994c2SFelix Kuehling #define SVM_IB_BASE (SVM_CWSR_BASE - PAGE_SIZE) 314d01994c2SFelix Kuehling 31570a31d16SFelix Kuehling static void kfd_init_apertures_vi(struct kfd_process_device *pdd, uint8_t id) 31670a31d16SFelix Kuehling { 31770a31d16SFelix Kuehling /* 31870a31d16SFelix Kuehling * node id couldn't be 0 - the three MSB bits of 31970a31d16SFelix Kuehling * aperture shoudn't be 0 32070a31d16SFelix Kuehling */ 32170a31d16SFelix Kuehling pdd->lds_base = MAKE_LDS_APP_BASE_VI(); 32270a31d16SFelix Kuehling pdd->lds_limit = MAKE_LDS_APP_LIMIT(pdd->lds_base); 32370a31d16SFelix Kuehling 32470a31d16SFelix Kuehling if (!pdd->dev->device_info->needs_iommu_device) { 32570a31d16SFelix Kuehling /* dGPUs: SVM aperture starting at 0 32670a31d16SFelix Kuehling * with small reserved space for kernel. 32770a31d16SFelix Kuehling * Set them to CANONICAL addresses. 32870a31d16SFelix Kuehling */ 32970a31d16SFelix Kuehling pdd->gpuvm_base = SVM_USER_BASE; 33070a31d16SFelix Kuehling pdd->gpuvm_limit = 33170a31d16SFelix Kuehling pdd->dev->shared_resources.gpuvm_size - 1; 33270a31d16SFelix Kuehling } else { 33370a31d16SFelix Kuehling /* set them to non CANONICAL addresses, and no SVM is 33470a31d16SFelix Kuehling * allocated. 33570a31d16SFelix Kuehling */ 33670a31d16SFelix Kuehling pdd->gpuvm_base = MAKE_GPUVM_APP_BASE_VI(id + 1); 33770a31d16SFelix Kuehling pdd->gpuvm_limit = MAKE_GPUVM_APP_LIMIT(pdd->gpuvm_base, 33870a31d16SFelix Kuehling pdd->dev->shared_resources.gpuvm_size); 33970a31d16SFelix Kuehling } 34070a31d16SFelix Kuehling 34170a31d16SFelix Kuehling pdd->scratch_base = MAKE_SCRATCH_APP_BASE_VI(); 34270a31d16SFelix Kuehling pdd->scratch_limit = MAKE_SCRATCH_APP_LIMIT(pdd->scratch_base); 34370a31d16SFelix Kuehling } 34470a31d16SFelix Kuehling 34570a31d16SFelix Kuehling static void kfd_init_apertures_v9(struct kfd_process_device *pdd, uint8_t id) 34670a31d16SFelix Kuehling { 34770a31d16SFelix Kuehling pdd->lds_base = MAKE_LDS_APP_BASE_V9(); 34870a31d16SFelix Kuehling pdd->lds_limit = MAKE_LDS_APP_LIMIT(pdd->lds_base); 34970a31d16SFelix Kuehling 35070a31d16SFelix Kuehling /* Raven needs SVM to support graphic handle, etc. Leave the small 35170a31d16SFelix Kuehling * reserved space before SVM on Raven as well, even though we don't 35270a31d16SFelix Kuehling * have to. 35370a31d16SFelix Kuehling * Set gpuvm_base and gpuvm_limit to CANONICAL addresses so that they 35470a31d16SFelix Kuehling * are used in Thunk to reserve SVM. 35570a31d16SFelix Kuehling */ 35670a31d16SFelix Kuehling pdd->gpuvm_base = SVM_USER_BASE; 35770a31d16SFelix Kuehling pdd->gpuvm_limit = 35870a31d16SFelix Kuehling pdd->dev->shared_resources.gpuvm_size - 1; 35970a31d16SFelix Kuehling 36070a31d16SFelix Kuehling pdd->scratch_base = MAKE_SCRATCH_APP_BASE_V9(); 36170a31d16SFelix Kuehling pdd->scratch_limit = MAKE_SCRATCH_APP_LIMIT(pdd->scratch_base); 36270a31d16SFelix Kuehling } 36370a31d16SFelix Kuehling 36419f6d2a6SOded Gabbay int kfd_init_apertures(struct kfd_process *process) 36519f6d2a6SOded Gabbay { 36619f6d2a6SOded Gabbay uint8_t id = 0; 36719f6d2a6SOded Gabbay struct kfd_dev *dev; 36819f6d2a6SOded Gabbay struct kfd_process_device *pdd; 36919f6d2a6SOded Gabbay 37019f6d2a6SOded Gabbay /*Iterating over all devices*/ 3716aac0a48SFelix Kuehling while (kfd_topology_enum_kfd_devices(id, &dev) == 0) { 3726d82eb0eSHarish Kasiviswanathan if (!dev) { 3736d82eb0eSHarish Kasiviswanathan id++; /* Skip non GPU devices */ 3746d82eb0eSHarish Kasiviswanathan continue; 3756d82eb0eSHarish Kasiviswanathan } 3766d82eb0eSHarish Kasiviswanathan 377093c7d8cSAlexey Skidanov pdd = kfd_create_process_device_data(dev, process); 3784eacc26bSKent Russell if (!pdd) { 379093c7d8cSAlexey Skidanov pr_err("Failed to create process device data\n"); 38070a31d16SFelix Kuehling return -ENOMEM; 381093c7d8cSAlexey Skidanov } 38219f6d2a6SOded Gabbay /* 383c7bcbfa4SFelix Kuehling * For 64 bit process apertures will be statically reserved in 38419f6d2a6SOded Gabbay * the x86_64 non canonical process address space 38519f6d2a6SOded Gabbay * amdkfd doesn't currently support apertures for 32 bit process 38619f6d2a6SOded Gabbay */ 38719f6d2a6SOded Gabbay if (process->is_32bit_user_mode) { 38819f6d2a6SOded Gabbay pdd->lds_base = pdd->lds_limit = 0; 38919f6d2a6SOded Gabbay pdd->gpuvm_base = pdd->gpuvm_limit = 0; 39019f6d2a6SOded Gabbay pdd->scratch_base = pdd->scratch_limit = 0; 39119f6d2a6SOded Gabbay } else { 39270a31d16SFelix Kuehling switch (dev->device_info->asic_family) { 39370a31d16SFelix Kuehling case CHIP_KAVERI: 39470a31d16SFelix Kuehling case CHIP_HAWAII: 39570a31d16SFelix Kuehling case CHIP_CARRIZO: 39670a31d16SFelix Kuehling case CHIP_TONGA: 39770a31d16SFelix Kuehling case CHIP_FIJI: 39870a31d16SFelix Kuehling case CHIP_POLARIS10: 39970a31d16SFelix Kuehling case CHIP_POLARIS11: 40070a31d16SFelix Kuehling kfd_init_apertures_vi(pdd, id); 40170a31d16SFelix Kuehling break; 40270a31d16SFelix Kuehling case CHIP_VEGA10: 40322a3a294SShaoyun Liu case CHIP_VEGA20: 40470a31d16SFelix Kuehling case CHIP_RAVEN: 40570a31d16SFelix Kuehling kfd_init_apertures_v9(pdd, id); 40670a31d16SFelix Kuehling break; 40770a31d16SFelix Kuehling default: 40870a31d16SFelix Kuehling WARN(1, "Unexpected ASIC family %u", 40970a31d16SFelix Kuehling dev->device_info->asic_family); 41070a31d16SFelix Kuehling return -EINVAL; 41170a31d16SFelix Kuehling } 41219f6d2a6SOded Gabbay 41370a31d16SFelix Kuehling if (!dev->device_info->needs_iommu_device) { 41470a31d16SFelix Kuehling /* dGPUs: the reserved space for kernel 41570a31d16SFelix Kuehling * before SVM 416d01994c2SFelix Kuehling */ 417d01994c2SFelix Kuehling pdd->qpd.cwsr_base = SVM_CWSR_BASE; 418d01994c2SFelix Kuehling pdd->qpd.ib_base = SVM_IB_BASE; 419d01994c2SFelix Kuehling } 42019f6d2a6SOded Gabbay } 42119f6d2a6SOded Gabbay 42219f6d2a6SOded Gabbay dev_dbg(kfd_device, "node id %u\n", id); 42319f6d2a6SOded Gabbay dev_dbg(kfd_device, "gpu id %u\n", pdd->dev->id); 42419f6d2a6SOded Gabbay dev_dbg(kfd_device, "lds_base %llX\n", pdd->lds_base); 42519f6d2a6SOded Gabbay dev_dbg(kfd_device, "lds_limit %llX\n", pdd->lds_limit); 42619f6d2a6SOded Gabbay dev_dbg(kfd_device, "gpuvm_base %llX\n", pdd->gpuvm_base); 42719f6d2a6SOded Gabbay dev_dbg(kfd_device, "gpuvm_limit %llX\n", pdd->gpuvm_limit); 42819f6d2a6SOded Gabbay dev_dbg(kfd_device, "scratch_base %llX\n", pdd->scratch_base); 42919f6d2a6SOded Gabbay dev_dbg(kfd_device, "scratch_limit %llX\n", pdd->scratch_limit); 43019f6d2a6SOded Gabbay 43119f6d2a6SOded Gabbay id++; 43219f6d2a6SOded Gabbay } 43319f6d2a6SOded Gabbay 43419f6d2a6SOded Gabbay return 0; 43519f6d2a6SOded Gabbay } 43619f6d2a6SOded Gabbay 43719f6d2a6SOded Gabbay 438