18672103fSJakub Kicinski /* 28672103fSJakub Kicinski * Copyright (C) 2015-2017 Netronome Systems, Inc. 38672103fSJakub Kicinski * 48672103fSJakub Kicinski * This software is dual licensed under the GNU General License Version 2, 58672103fSJakub Kicinski * June 1991 as shown in the file COPYING in the top-level directory of this 68672103fSJakub Kicinski * source tree or the BSD 2-Clause License provided below. You have the 78672103fSJakub Kicinski * option to license this software under the complete terms of either license. 88672103fSJakub Kicinski * 98672103fSJakub Kicinski * The BSD 2-Clause License: 108672103fSJakub Kicinski * 118672103fSJakub Kicinski * Redistribution and use in source and binary forms, with or 128672103fSJakub Kicinski * without modification, are permitted provided that the following 138672103fSJakub Kicinski * conditions are met: 148672103fSJakub Kicinski * 158672103fSJakub Kicinski * 1. Redistributions of source code must retain the above 168672103fSJakub Kicinski * copyright notice, this list of conditions and the following 178672103fSJakub Kicinski * disclaimer. 188672103fSJakub Kicinski * 198672103fSJakub Kicinski * 2. Redistributions in binary form must reproduce the above 208672103fSJakub Kicinski * copyright notice, this list of conditions and the following 218672103fSJakub Kicinski * disclaimer in the documentation and/or other materials 228672103fSJakub Kicinski * provided with the distribution. 238672103fSJakub Kicinski * 248672103fSJakub Kicinski * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 258672103fSJakub Kicinski * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 268672103fSJakub Kicinski * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 278672103fSJakub Kicinski * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 288672103fSJakub Kicinski * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 298672103fSJakub Kicinski * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 308672103fSJakub Kicinski * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 318672103fSJakub Kicinski * SOFTWARE. 328672103fSJakub Kicinski */ 338672103fSJakub Kicinski 348672103fSJakub Kicinski #include <linux/delay.h> 358672103fSJakub Kicinski #include <linux/device.h> 368672103fSJakub Kicinski #include <linux/jiffies.h> 378672103fSJakub Kicinski #include <linux/types.h> 388672103fSJakub Kicinski #include <linux/slab.h> 398672103fSJakub Kicinski #include <linux/wait.h> 408672103fSJakub Kicinski 418672103fSJakub Kicinski #include "nfp_cpp.h" 428672103fSJakub Kicinski #include "nfp6000/nfp6000.h" 438672103fSJakub Kicinski 448672103fSJakub Kicinski struct nfp_cpp_mutex { 458672103fSJakub Kicinski struct nfp_cpp *cpp; 468672103fSJakub Kicinski int target; 478672103fSJakub Kicinski u16 depth; 488672103fSJakub Kicinski unsigned long long address; 498672103fSJakub Kicinski u32 key; 508672103fSJakub Kicinski }; 518672103fSJakub Kicinski 528672103fSJakub Kicinski static u32 nfp_mutex_locked(u16 interface) 538672103fSJakub Kicinski { 548672103fSJakub Kicinski return (u32)interface << 16 | 0x000f; 558672103fSJakub Kicinski } 568672103fSJakub Kicinski 578672103fSJakub Kicinski static u32 nfp_mutex_unlocked(u16 interface) 588672103fSJakub Kicinski { 598672103fSJakub Kicinski return (u32)interface << 16 | 0x0000; 608672103fSJakub Kicinski } 618672103fSJakub Kicinski 628672103fSJakub Kicinski static bool nfp_mutex_is_locked(u32 val) 638672103fSJakub Kicinski { 648672103fSJakub Kicinski return (val & 0xffff) == 0x000f; 658672103fSJakub Kicinski } 668672103fSJakub Kicinski 678672103fSJakub Kicinski static bool nfp_mutex_is_unlocked(u32 val) 688672103fSJakub Kicinski { 698672103fSJakub Kicinski return (val & 0xffff) == 0000; 708672103fSJakub Kicinski } 718672103fSJakub Kicinski 728672103fSJakub Kicinski /* If you need more than 65536 recursive locks, please rethink your code. */ 738672103fSJakub Kicinski #define NFP_MUTEX_DEPTH_MAX 0xffff 748672103fSJakub Kicinski 758672103fSJakub Kicinski static int 768672103fSJakub Kicinski nfp_cpp_mutex_validate(u16 interface, int *target, unsigned long long address) 778672103fSJakub Kicinski { 788672103fSJakub Kicinski /* Not permitted on invalid interfaces */ 798672103fSJakub Kicinski if (NFP_CPP_INTERFACE_TYPE_of(interface) == 808672103fSJakub Kicinski NFP_CPP_INTERFACE_TYPE_INVALID) 818672103fSJakub Kicinski return -EINVAL; 828672103fSJakub Kicinski 838672103fSJakub Kicinski /* Address must be 64-bit aligned */ 848672103fSJakub Kicinski if (address & 7) 858672103fSJakub Kicinski return -EINVAL; 868672103fSJakub Kicinski 878672103fSJakub Kicinski if (*target != NFP_CPP_TARGET_MU) 888672103fSJakub Kicinski return -EINVAL; 898672103fSJakub Kicinski 908672103fSJakub Kicinski return 0; 918672103fSJakub Kicinski } 928672103fSJakub Kicinski 938672103fSJakub Kicinski /** 948672103fSJakub Kicinski * nfp_cpp_mutex_init() - Initialize a mutex location 958672103fSJakub Kicinski * @cpp: NFP CPP handle 968672103fSJakub Kicinski * @target: NFP CPP target ID (ie NFP_CPP_TARGET_CLS or NFP_CPP_TARGET_MU) 978672103fSJakub Kicinski * @address: Offset into the address space of the NFP CPP target ID 988672103fSJakub Kicinski * @key: Unique 32-bit value for this mutex 998672103fSJakub Kicinski * 1008672103fSJakub Kicinski * The CPP target:address must point to a 64-bit aligned location, and 1018672103fSJakub Kicinski * will initialize 64 bits of data at the location. 1028672103fSJakub Kicinski * 1038672103fSJakub Kicinski * This creates the initial mutex state, as locked by this 1048672103fSJakub Kicinski * nfp_cpp_interface(). 1058672103fSJakub Kicinski * 1068672103fSJakub Kicinski * This function should only be called when setting up 1078672103fSJakub Kicinski * the initial lock state upon boot-up of the system. 1088672103fSJakub Kicinski * 1098672103fSJakub Kicinski * Return: 0 on success, or -errno on failure 1108672103fSJakub Kicinski */ 1118672103fSJakub Kicinski int nfp_cpp_mutex_init(struct nfp_cpp *cpp, 1128672103fSJakub Kicinski int target, unsigned long long address, u32 key) 1138672103fSJakub Kicinski { 1148672103fSJakub Kicinski const u32 muw = NFP_CPP_ID(target, 4, 0); /* atomic_write */ 1158672103fSJakub Kicinski u16 interface = nfp_cpp_interface(cpp); 1168672103fSJakub Kicinski int err; 1178672103fSJakub Kicinski 1188672103fSJakub Kicinski err = nfp_cpp_mutex_validate(interface, &target, address); 1198672103fSJakub Kicinski if (err) 1208672103fSJakub Kicinski return err; 1218672103fSJakub Kicinski 1228672103fSJakub Kicinski err = nfp_cpp_writel(cpp, muw, address + 4, key); 1238672103fSJakub Kicinski if (err) 1248672103fSJakub Kicinski return err; 1258672103fSJakub Kicinski 1268672103fSJakub Kicinski err = nfp_cpp_writel(cpp, muw, address, nfp_mutex_locked(interface)); 1278672103fSJakub Kicinski if (err) 1288672103fSJakub Kicinski return err; 1298672103fSJakub Kicinski 1308672103fSJakub Kicinski return 0; 1318672103fSJakub Kicinski } 1328672103fSJakub Kicinski 1338672103fSJakub Kicinski /** 1348672103fSJakub Kicinski * nfp_cpp_mutex_alloc() - Create a mutex handle 1358672103fSJakub Kicinski * @cpp: NFP CPP handle 1368672103fSJakub Kicinski * @target: NFP CPP target ID (ie NFP_CPP_TARGET_CLS or NFP_CPP_TARGET_MU) 1378672103fSJakub Kicinski * @address: Offset into the address space of the NFP CPP target ID 1388672103fSJakub Kicinski * @key: 32-bit unique key (must match the key at this location) 1398672103fSJakub Kicinski * 1408672103fSJakub Kicinski * The CPP target:address must point to a 64-bit aligned location, and 1418672103fSJakub Kicinski * reserve 64 bits of data at the location for use by the handle. 1428672103fSJakub Kicinski * 1438672103fSJakub Kicinski * Only target/address pairs that point to entities that support the 1448672103fSJakub Kicinski * MU Atomic Engine's CmpAndSwap32 command are supported. 1458672103fSJakub Kicinski * 1468672103fSJakub Kicinski * Return: A non-NULL struct nfp_cpp_mutex * on success, NULL on failure. 1478672103fSJakub Kicinski */ 1488672103fSJakub Kicinski struct nfp_cpp_mutex *nfp_cpp_mutex_alloc(struct nfp_cpp *cpp, int target, 1498672103fSJakub Kicinski unsigned long long address, u32 key) 1508672103fSJakub Kicinski { 1518672103fSJakub Kicinski const u32 mur = NFP_CPP_ID(target, 3, 0); /* atomic_read */ 1528672103fSJakub Kicinski u16 interface = nfp_cpp_interface(cpp); 1538672103fSJakub Kicinski struct nfp_cpp_mutex *mutex; 1548672103fSJakub Kicinski int err; 1558672103fSJakub Kicinski u32 tmp; 1568672103fSJakub Kicinski 1578672103fSJakub Kicinski err = nfp_cpp_mutex_validate(interface, &target, address); 1588672103fSJakub Kicinski if (err) 1598672103fSJakub Kicinski return NULL; 1608672103fSJakub Kicinski 1618672103fSJakub Kicinski err = nfp_cpp_readl(cpp, mur, address + 4, &tmp); 1628672103fSJakub Kicinski if (err < 0) 1638672103fSJakub Kicinski return NULL; 1648672103fSJakub Kicinski 1658672103fSJakub Kicinski if (tmp != key) 1668672103fSJakub Kicinski return NULL; 1678672103fSJakub Kicinski 1688672103fSJakub Kicinski mutex = kzalloc(sizeof(*mutex), GFP_KERNEL); 1698672103fSJakub Kicinski if (!mutex) 1708672103fSJakub Kicinski return NULL; 1718672103fSJakub Kicinski 1728672103fSJakub Kicinski mutex->cpp = cpp; 1738672103fSJakub Kicinski mutex->target = target; 1748672103fSJakub Kicinski mutex->address = address; 1758672103fSJakub Kicinski mutex->key = key; 1768672103fSJakub Kicinski mutex->depth = 0; 1778672103fSJakub Kicinski 1788672103fSJakub Kicinski return mutex; 1798672103fSJakub Kicinski } 1808672103fSJakub Kicinski 1818672103fSJakub Kicinski /** 1828672103fSJakub Kicinski * nfp_cpp_mutex_free() - Free a mutex handle - does not alter the lock state 1838672103fSJakub Kicinski * @mutex: NFP CPP Mutex handle 1848672103fSJakub Kicinski */ 1858672103fSJakub Kicinski void nfp_cpp_mutex_free(struct nfp_cpp_mutex *mutex) 1868672103fSJakub Kicinski { 1878672103fSJakub Kicinski kfree(mutex); 1888672103fSJakub Kicinski } 1898672103fSJakub Kicinski 1908672103fSJakub Kicinski /** 1918672103fSJakub Kicinski * nfp_cpp_mutex_lock() - Lock a mutex handle, using the NFP MU Atomic Engine 1928672103fSJakub Kicinski * @mutex: NFP CPP Mutex handle 1938672103fSJakub Kicinski * 1948672103fSJakub Kicinski * Return: 0 on success, or -errno on failure 1958672103fSJakub Kicinski */ 1968672103fSJakub Kicinski int nfp_cpp_mutex_lock(struct nfp_cpp_mutex *mutex) 1978672103fSJakub Kicinski { 1988672103fSJakub Kicinski unsigned long warn_at = jiffies + 15 * HZ; 1998672103fSJakub Kicinski unsigned int timeout_ms = 1; 2008672103fSJakub Kicinski int err; 2018672103fSJakub Kicinski 2028672103fSJakub Kicinski /* We can't use a waitqueue here, because the unlocker 2038672103fSJakub Kicinski * might be on a separate CPU. 2048672103fSJakub Kicinski * 2058672103fSJakub Kicinski * So just wait for now. 2068672103fSJakub Kicinski */ 2078672103fSJakub Kicinski for (;;) { 2088672103fSJakub Kicinski err = nfp_cpp_mutex_trylock(mutex); 2098672103fSJakub Kicinski if (err != -EBUSY) 2108672103fSJakub Kicinski break; 2118672103fSJakub Kicinski 2128672103fSJakub Kicinski err = msleep_interruptible(timeout_ms); 2138672103fSJakub Kicinski if (err != 0) 2148672103fSJakub Kicinski return -ERESTARTSYS; 2158672103fSJakub Kicinski 2168672103fSJakub Kicinski if (time_is_before_eq_jiffies(warn_at)) { 2178672103fSJakub Kicinski warn_at = jiffies + 60 * HZ; 2188672103fSJakub Kicinski nfp_warn(mutex->cpp, 2198672103fSJakub Kicinski "Warning: waiting for NFP mutex [depth:%hd target:%d addr:%llx key:%08x]\n", 2208672103fSJakub Kicinski mutex->depth, 2218672103fSJakub Kicinski mutex->target, mutex->address, mutex->key); 2228672103fSJakub Kicinski } 2238672103fSJakub Kicinski } 2248672103fSJakub Kicinski 2258672103fSJakub Kicinski return err; 2268672103fSJakub Kicinski } 2278672103fSJakub Kicinski 2288672103fSJakub Kicinski /** 2298672103fSJakub Kicinski * nfp_cpp_mutex_unlock() - Unlock a mutex handle, using the MU Atomic Engine 2308672103fSJakub Kicinski * @mutex: NFP CPP Mutex handle 2318672103fSJakub Kicinski * 2328672103fSJakub Kicinski * Return: 0 on success, or -errno on failure 2338672103fSJakub Kicinski */ 2348672103fSJakub Kicinski int nfp_cpp_mutex_unlock(struct nfp_cpp_mutex *mutex) 2358672103fSJakub Kicinski { 2368672103fSJakub Kicinski const u32 muw = NFP_CPP_ID(mutex->target, 4, 0); /* atomic_write */ 2378672103fSJakub Kicinski const u32 mur = NFP_CPP_ID(mutex->target, 3, 0); /* atomic_read */ 2388672103fSJakub Kicinski struct nfp_cpp *cpp = mutex->cpp; 2398672103fSJakub Kicinski u32 key, value; 2408672103fSJakub Kicinski u16 interface; 2418672103fSJakub Kicinski int err; 2428672103fSJakub Kicinski 2438672103fSJakub Kicinski interface = nfp_cpp_interface(cpp); 2448672103fSJakub Kicinski 2458672103fSJakub Kicinski if (mutex->depth > 1) { 2468672103fSJakub Kicinski mutex->depth--; 2478672103fSJakub Kicinski return 0; 2488672103fSJakub Kicinski } 2498672103fSJakub Kicinski 2508672103fSJakub Kicinski err = nfp_cpp_readl(mutex->cpp, mur, mutex->address + 4, &key); 2518672103fSJakub Kicinski if (err < 0) 2528672103fSJakub Kicinski return err; 2538672103fSJakub Kicinski 2548672103fSJakub Kicinski if (key != mutex->key) 2558672103fSJakub Kicinski return -EPERM; 2568672103fSJakub Kicinski 2578672103fSJakub Kicinski err = nfp_cpp_readl(mutex->cpp, mur, mutex->address, &value); 2588672103fSJakub Kicinski if (err < 0) 2598672103fSJakub Kicinski return err; 2608672103fSJakub Kicinski 2618672103fSJakub Kicinski if (value != nfp_mutex_locked(interface)) 2628672103fSJakub Kicinski return -EACCES; 2638672103fSJakub Kicinski 2648672103fSJakub Kicinski err = nfp_cpp_writel(cpp, muw, mutex->address, 2658672103fSJakub Kicinski nfp_mutex_unlocked(interface)); 2668672103fSJakub Kicinski if (err < 0) 2678672103fSJakub Kicinski return err; 2688672103fSJakub Kicinski 2698672103fSJakub Kicinski mutex->depth = 0; 2708672103fSJakub Kicinski return 0; 2718672103fSJakub Kicinski } 2728672103fSJakub Kicinski 2738672103fSJakub Kicinski /** 2748672103fSJakub Kicinski * nfp_cpp_mutex_trylock() - Attempt to lock a mutex handle 2758672103fSJakub Kicinski * @mutex: NFP CPP Mutex handle 2768672103fSJakub Kicinski * 2778672103fSJakub Kicinski * Return: 0 if the lock succeeded, -errno on failure 2788672103fSJakub Kicinski */ 2798672103fSJakub Kicinski int nfp_cpp_mutex_trylock(struct nfp_cpp_mutex *mutex) 2808672103fSJakub Kicinski { 2818672103fSJakub Kicinski const u32 muw = NFP_CPP_ID(mutex->target, 4, 0); /* atomic_write */ 2828672103fSJakub Kicinski const u32 mus = NFP_CPP_ID(mutex->target, 5, 3); /* test_set_imm */ 2838672103fSJakub Kicinski const u32 mur = NFP_CPP_ID(mutex->target, 3, 0); /* atomic_read */ 2848672103fSJakub Kicinski struct nfp_cpp *cpp = mutex->cpp; 2858672103fSJakub Kicinski u32 key, value, tmp; 2868672103fSJakub Kicinski int err; 2878672103fSJakub Kicinski 2888672103fSJakub Kicinski if (mutex->depth > 0) { 2898672103fSJakub Kicinski if (mutex->depth == NFP_MUTEX_DEPTH_MAX) 2908672103fSJakub Kicinski return -E2BIG; 2918672103fSJakub Kicinski mutex->depth++; 2928672103fSJakub Kicinski return 0; 2938672103fSJakub Kicinski } 2948672103fSJakub Kicinski 2958672103fSJakub Kicinski /* Verify that the lock marker is not damaged */ 2968672103fSJakub Kicinski err = nfp_cpp_readl(cpp, mur, mutex->address + 4, &key); 2978672103fSJakub Kicinski if (err < 0) 2988672103fSJakub Kicinski return err; 2998672103fSJakub Kicinski 3008672103fSJakub Kicinski if (key != mutex->key) 3018672103fSJakub Kicinski return -EPERM; 3028672103fSJakub Kicinski 3038672103fSJakub Kicinski /* Compare against the unlocked state, and if true, 3048672103fSJakub Kicinski * write the interface id into the top 16 bits, and 3058672103fSJakub Kicinski * mark as locked. 3068672103fSJakub Kicinski */ 3078672103fSJakub Kicinski value = nfp_mutex_locked(nfp_cpp_interface(cpp)); 3088672103fSJakub Kicinski 3098672103fSJakub Kicinski /* We use test_set_imm here, as it implies a read 3108672103fSJakub Kicinski * of the current state, and sets the bits in the 3118672103fSJakub Kicinski * bytemask of the command to 1s. Since the mutex 3128672103fSJakub Kicinski * is guaranteed to be 64-bit aligned, the bytemask 3138672103fSJakub Kicinski * of this 32-bit command is ensured to be 8'b00001111, 3148672103fSJakub Kicinski * which implies that the lower 4 bits will be set to 3158672103fSJakub Kicinski * ones regardless of the initial state. 3168672103fSJakub Kicinski * 3178672103fSJakub Kicinski * Since this is a 'Readback' operation, with no Pull 3188672103fSJakub Kicinski * data, we can treat this as a normal Push (read) 3198672103fSJakub Kicinski * atomic, which returns the original value. 3208672103fSJakub Kicinski */ 3218672103fSJakub Kicinski err = nfp_cpp_readl(cpp, mus, mutex->address, &tmp); 3228672103fSJakub Kicinski if (err < 0) 3238672103fSJakub Kicinski return err; 3248672103fSJakub Kicinski 3258672103fSJakub Kicinski /* Was it unlocked? */ 3268672103fSJakub Kicinski if (nfp_mutex_is_unlocked(tmp)) { 3278672103fSJakub Kicinski /* The read value can only be 0x....0000 in the unlocked state. 3288672103fSJakub Kicinski * If there was another contending for this lock, then 3298672103fSJakub Kicinski * the lock state would be 0x....000f 3308672103fSJakub Kicinski */ 3318672103fSJakub Kicinski 3328672103fSJakub Kicinski /* Write our owner ID into the lock 3338672103fSJakub Kicinski * While not strictly necessary, this helps with 3348672103fSJakub Kicinski * debug and bookkeeping. 3358672103fSJakub Kicinski */ 3368672103fSJakub Kicinski err = nfp_cpp_writel(cpp, muw, mutex->address, value); 3378672103fSJakub Kicinski if (err < 0) 3388672103fSJakub Kicinski return err; 3398672103fSJakub Kicinski 3408672103fSJakub Kicinski mutex->depth = 1; 3418672103fSJakub Kicinski return 0; 3428672103fSJakub Kicinski } 3438672103fSJakub Kicinski 3448672103fSJakub Kicinski return nfp_mutex_is_locked(tmp) ? -EBUSY : -EINVAL; 3458672103fSJakub Kicinski } 346