1f033c26dSMark Brown // SPDX-License-Identifier: GPL-2.0 2f033c26dSMark Brown // 3f033c26dSMark Brown // Register cache access API - maple tree based cache 4f033c26dSMark Brown // 5f033c26dSMark Brown // Copyright 2023 Arm, Ltd 6f033c26dSMark Brown // 7f033c26dSMark Brown // Author: Mark Brown <broonie@kernel.org> 8f033c26dSMark Brown 9f033c26dSMark Brown #include <linux/debugfs.h> 10f033c26dSMark Brown #include <linux/device.h> 11f033c26dSMark Brown #include <linux/maple_tree.h> 12f033c26dSMark Brown #include <linux/slab.h> 13f033c26dSMark Brown 14f033c26dSMark Brown #include "internal.h" 15f033c26dSMark Brown 16f033c26dSMark Brown static int regcache_maple_read(struct regmap *map, 17f033c26dSMark Brown unsigned int reg, unsigned int *value) 18f033c26dSMark Brown { 19f033c26dSMark Brown struct maple_tree *mt = map->cache; 20f033c26dSMark Brown MA_STATE(mas, mt, reg, reg); 21f033c26dSMark Brown unsigned long *entry; 22f033c26dSMark Brown 23f033c26dSMark Brown rcu_read_lock(); 24f033c26dSMark Brown 25fac79badSMark Brown entry = mas_walk(&mas); 26f033c26dSMark Brown if (!entry) { 27f033c26dSMark Brown rcu_read_unlock(); 28f033c26dSMark Brown return -ENOENT; 29f033c26dSMark Brown } 30f033c26dSMark Brown 31f033c26dSMark Brown *value = entry[reg - mas.index]; 32f033c26dSMark Brown 33f033c26dSMark Brown rcu_read_unlock(); 34f033c26dSMark Brown 35f033c26dSMark Brown return 0; 36f033c26dSMark Brown } 37f033c26dSMark Brown 38f033c26dSMark Brown static int regcache_maple_write(struct regmap *map, unsigned int reg, 39f033c26dSMark Brown unsigned int val) 40f033c26dSMark Brown { 41f033c26dSMark Brown struct maple_tree *mt = map->cache; 42f033c26dSMark Brown MA_STATE(mas, mt, reg, reg); 43f033c26dSMark Brown unsigned long *entry, *upper, *lower; 44f033c26dSMark Brown unsigned long index, last; 45f033c26dSMark Brown size_t lower_sz, upper_sz; 46f033c26dSMark Brown int ret; 47f033c26dSMark Brown 48f033c26dSMark Brown rcu_read_lock(); 49f033c26dSMark Brown 50fac79badSMark Brown entry = mas_walk(&mas); 51f033c26dSMark Brown if (entry) { 52f033c26dSMark Brown entry[reg - mas.index] = val; 53f033c26dSMark Brown rcu_read_unlock(); 54f033c26dSMark Brown return 0; 55f033c26dSMark Brown } 56f033c26dSMark Brown 57f033c26dSMark Brown /* Any adjacent entries to extend/merge? */ 58f033c26dSMark Brown mas_set_range(&mas, reg - 1, reg + 1); 59f033c26dSMark Brown index = reg; 60f033c26dSMark Brown last = reg; 61f033c26dSMark Brown 62f033c26dSMark Brown lower = mas_find(&mas, reg - 1); 63f033c26dSMark Brown if (lower) { 64f033c26dSMark Brown index = mas.index; 65f033c26dSMark Brown lower_sz = (mas.last - mas.index + 1) * sizeof(unsigned long); 66f033c26dSMark Brown } 67f033c26dSMark Brown 68f033c26dSMark Brown upper = mas_find(&mas, reg + 1); 69f033c26dSMark Brown if (upper) { 70f033c26dSMark Brown last = mas.last; 71f033c26dSMark Brown upper_sz = (mas.last - mas.index + 1) * sizeof(unsigned long); 72f033c26dSMark Brown } 73f033c26dSMark Brown 74f033c26dSMark Brown rcu_read_unlock(); 75f033c26dSMark Brown 76f033c26dSMark Brown entry = kmalloc((last - index + 1) * sizeof(unsigned long), 77f033c26dSMark Brown GFP_KERNEL); 78f033c26dSMark Brown if (!entry) 79f033c26dSMark Brown return -ENOMEM; 80f033c26dSMark Brown 81f033c26dSMark Brown if (lower) 82f033c26dSMark Brown memcpy(entry, lower, lower_sz); 83f033c26dSMark Brown entry[reg - index] = val; 84f033c26dSMark Brown if (upper) 85f033c26dSMark Brown memcpy(&entry[reg - index + 1], upper, upper_sz); 86f033c26dSMark Brown 87f033c26dSMark Brown /* 88f033c26dSMark Brown * This is safe because the regmap lock means the Maple lock 89f033c26dSMark Brown * is redundant, but we need to take it due to lockdep asserts 90f033c26dSMark Brown * in the maple tree code. 91f033c26dSMark Brown */ 92f033c26dSMark Brown mas_lock(&mas); 93f033c26dSMark Brown 94f033c26dSMark Brown mas_set_range(&mas, index, last); 95f033c26dSMark Brown ret = mas_store_gfp(&mas, entry, GFP_KERNEL); 96f033c26dSMark Brown 97f033c26dSMark Brown mas_unlock(&mas); 98f033c26dSMark Brown 99f033c26dSMark Brown if (ret == 0) { 100f033c26dSMark Brown kfree(lower); 101f033c26dSMark Brown kfree(upper); 102f033c26dSMark Brown } 103f033c26dSMark Brown 104f033c26dSMark Brown return ret; 105f033c26dSMark Brown } 106f033c26dSMark Brown 107f033c26dSMark Brown static int regcache_maple_drop(struct regmap *map, unsigned int min, 108f033c26dSMark Brown unsigned int max) 109f033c26dSMark Brown { 110f033c26dSMark Brown struct maple_tree *mt = map->cache; 111f033c26dSMark Brown MA_STATE(mas, mt, min, max); 112f033c26dSMark Brown unsigned long *entry, *lower, *upper; 113f033c26dSMark Brown unsigned long lower_index, lower_last; 114f033c26dSMark Brown unsigned long upper_index, upper_last; 115f033c26dSMark Brown int ret; 116f033c26dSMark Brown 117f033c26dSMark Brown lower = NULL; 118f033c26dSMark Brown upper = NULL; 119f033c26dSMark Brown 120f033c26dSMark Brown mas_lock(&mas); 121f033c26dSMark Brown 122f033c26dSMark Brown mas_for_each(&mas, entry, max) { 123f033c26dSMark Brown /* 124f033c26dSMark Brown * This is safe because the regmap lock means the 125f033c26dSMark Brown * Maple lock is redundant, but we need to take it due 126f033c26dSMark Brown * to lockdep asserts in the maple tree code. 127f033c26dSMark Brown */ 128f033c26dSMark Brown mas_unlock(&mas); 129f033c26dSMark Brown 130f033c26dSMark Brown /* Do we need to save any of this entry? */ 131f033c26dSMark Brown if (mas.index < min) { 132f033c26dSMark Brown lower_index = mas.index; 133f033c26dSMark Brown lower_last = min -1; 134f033c26dSMark Brown 135f033c26dSMark Brown lower = kmemdup(entry, ((min - mas.index) * 136f033c26dSMark Brown sizeof(unsigned long)), 137f033c26dSMark Brown GFP_KERNEL); 138f033c26dSMark Brown if (!lower) { 139f033c26dSMark Brown ret = -ENOMEM; 140451941acSMark Brown goto out_unlocked; 141f033c26dSMark Brown } 142f033c26dSMark Brown } 143f033c26dSMark Brown 144f033c26dSMark Brown if (mas.last > max) { 145f033c26dSMark Brown upper_index = max + 1; 146f033c26dSMark Brown upper_last = mas.last; 147f033c26dSMark Brown 148f033c26dSMark Brown upper = kmemdup(&entry[max + 1], 149f033c26dSMark Brown ((mas.last - max) * 150f033c26dSMark Brown sizeof(unsigned long)), 151f033c26dSMark Brown GFP_KERNEL); 152f033c26dSMark Brown if (!upper) { 153f033c26dSMark Brown ret = -ENOMEM; 154451941acSMark Brown goto out_unlocked; 155f033c26dSMark Brown } 156f033c26dSMark Brown } 157f033c26dSMark Brown 158f033c26dSMark Brown kfree(entry); 159f033c26dSMark Brown mas_lock(&mas); 160f033c26dSMark Brown mas_erase(&mas); 161f033c26dSMark Brown 162f033c26dSMark Brown /* Insert new nodes with the saved data */ 163f033c26dSMark Brown if (lower) { 164f033c26dSMark Brown mas_set_range(&mas, lower_index, lower_last); 165f033c26dSMark Brown ret = mas_store_gfp(&mas, lower, GFP_KERNEL); 166f033c26dSMark Brown if (ret != 0) 167f033c26dSMark Brown goto out; 168f033c26dSMark Brown lower = NULL; 169f033c26dSMark Brown } 170f033c26dSMark Brown 171f033c26dSMark Brown if (upper) { 172f033c26dSMark Brown mas_set_range(&mas, upper_index, upper_last); 173f033c26dSMark Brown ret = mas_store_gfp(&mas, upper, GFP_KERNEL); 174f033c26dSMark Brown if (ret != 0) 175f033c26dSMark Brown goto out; 176f033c26dSMark Brown upper = NULL; 177f033c26dSMark Brown } 178f033c26dSMark Brown } 179f033c26dSMark Brown 180f033c26dSMark Brown out: 181f033c26dSMark Brown mas_unlock(&mas); 182451941acSMark Brown out_unlocked: 183f033c26dSMark Brown kfree(lower); 184f033c26dSMark Brown kfree(upper); 185f033c26dSMark Brown 186f033c26dSMark Brown return ret; 187f033c26dSMark Brown } 188f033c26dSMark Brown 189f033c26dSMark Brown static int regcache_maple_sync(struct regmap *map, unsigned int min, 190f033c26dSMark Brown unsigned int max) 191f033c26dSMark Brown { 192f033c26dSMark Brown struct maple_tree *mt = map->cache; 193f033c26dSMark Brown unsigned long *entry; 194f033c26dSMark Brown MA_STATE(mas, mt, min, max); 195f033c26dSMark Brown unsigned long lmin = min; 196f033c26dSMark Brown unsigned long lmax = max; 197f033c26dSMark Brown unsigned int r; 198f033c26dSMark Brown int ret; 199f033c26dSMark Brown 200f033c26dSMark Brown map->cache_bypass = true; 201f033c26dSMark Brown 202f033c26dSMark Brown rcu_read_lock(); 203f033c26dSMark Brown 204f033c26dSMark Brown mas_for_each(&mas, entry, max) { 205f033c26dSMark Brown for (r = max(mas.index, lmin); r <= min(mas.last, lmax); r++) { 206f033c26dSMark Brown ret = regcache_sync_val(map, r, entry[r - mas.index]); 207f033c26dSMark Brown if (ret != 0) 208f033c26dSMark Brown goto out; 209f033c26dSMark Brown } 210f033c26dSMark Brown } 211f033c26dSMark Brown 212f033c26dSMark Brown out: 213f033c26dSMark Brown rcu_read_unlock(); 214f033c26dSMark Brown 215f033c26dSMark Brown map->cache_bypass = false; 216f033c26dSMark Brown 217f033c26dSMark Brown return ret; 218f033c26dSMark Brown } 219f033c26dSMark Brown 220f033c26dSMark Brown static int regcache_maple_exit(struct regmap *map) 221f033c26dSMark Brown { 222f033c26dSMark Brown struct maple_tree *mt = map->cache; 223f033c26dSMark Brown MA_STATE(mas, mt, 0, UINT_MAX); 224f033c26dSMark Brown unsigned int *entry;; 225f033c26dSMark Brown 226f033c26dSMark Brown /* if we've already been called then just return */ 227f033c26dSMark Brown if (!mt) 228f033c26dSMark Brown return 0; 229f033c26dSMark Brown 230f033c26dSMark Brown mas_lock(&mas); 231f033c26dSMark Brown mas_for_each(&mas, entry, UINT_MAX) 232f033c26dSMark Brown kfree(entry); 233f033c26dSMark Brown __mt_destroy(mt); 234f033c26dSMark Brown mas_unlock(&mas); 235f033c26dSMark Brown 236f033c26dSMark Brown kfree(mt); 237f033c26dSMark Brown map->cache = NULL; 238f033c26dSMark Brown 239f033c26dSMark Brown return 0; 240f033c26dSMark Brown } 241f033c26dSMark Brown 242*3a48d212SMark Brown static int regcache_maple_insert_block(struct regmap *map, int first, 243*3a48d212SMark Brown int last) 244*3a48d212SMark Brown { 245*3a48d212SMark Brown struct maple_tree *mt = map->cache; 246*3a48d212SMark Brown MA_STATE(mas, mt, first, last); 247*3a48d212SMark Brown unsigned long *entry; 248*3a48d212SMark Brown int i, ret; 249*3a48d212SMark Brown 250*3a48d212SMark Brown entry = kcalloc(last - first + 1, sizeof(unsigned long), GFP_KERNEL); 251*3a48d212SMark Brown if (!entry) 252*3a48d212SMark Brown return -ENOMEM; 253*3a48d212SMark Brown 254*3a48d212SMark Brown for (i = 0; i < last - first + 1; i++) 255*3a48d212SMark Brown entry[i] = map->reg_defaults[first + i].def; 256*3a48d212SMark Brown 257*3a48d212SMark Brown mas_lock(&mas); 258*3a48d212SMark Brown 259*3a48d212SMark Brown mas_set_range(&mas, map->reg_defaults[first].reg, 260*3a48d212SMark Brown map->reg_defaults[last].reg); 261*3a48d212SMark Brown ret = mas_store_gfp(&mas, entry, GFP_KERNEL); 262*3a48d212SMark Brown 263*3a48d212SMark Brown mas_unlock(&mas); 264*3a48d212SMark Brown 265*3a48d212SMark Brown if (ret) 266*3a48d212SMark Brown kfree(entry); 267*3a48d212SMark Brown 268*3a48d212SMark Brown return ret; 269*3a48d212SMark Brown } 270*3a48d212SMark Brown 271f033c26dSMark Brown static int regcache_maple_init(struct regmap *map) 272f033c26dSMark Brown { 273f033c26dSMark Brown struct maple_tree *mt; 274f033c26dSMark Brown int i; 275f033c26dSMark Brown int ret; 276*3a48d212SMark Brown int range_start; 277f033c26dSMark Brown 278f033c26dSMark Brown mt = kmalloc(sizeof(*mt), GFP_KERNEL); 279f033c26dSMark Brown if (!mt) 280f033c26dSMark Brown return -ENOMEM; 281f033c26dSMark Brown map->cache = mt; 282f033c26dSMark Brown 283f033c26dSMark Brown mt_init(mt); 284f033c26dSMark Brown 285*3a48d212SMark Brown if (!map->num_reg_defaults) 286*3a48d212SMark Brown return 0; 287*3a48d212SMark Brown 288*3a48d212SMark Brown range_start = 0; 289*3a48d212SMark Brown 290*3a48d212SMark Brown /* Scan for ranges of contiguous registers */ 291*3a48d212SMark Brown for (i = 1; i < map->num_reg_defaults; i++) { 292*3a48d212SMark Brown if (map->reg_defaults[i].reg != 293*3a48d212SMark Brown map->reg_defaults[i - 1].reg + 1) { 294*3a48d212SMark Brown ret = regcache_maple_insert_block(map, range_start, 295*3a48d212SMark Brown i - 1); 296*3a48d212SMark Brown if (ret != 0) 297f033c26dSMark Brown goto err; 298*3a48d212SMark Brown 299*3a48d212SMark Brown range_start = i; 300f033c26dSMark Brown } 301*3a48d212SMark Brown } 302*3a48d212SMark Brown 303*3a48d212SMark Brown /* Add the last block */ 304*3a48d212SMark Brown ret = regcache_maple_insert_block(map, range_start, 305*3a48d212SMark Brown map->num_reg_defaults - 1); 306*3a48d212SMark Brown if (ret != 0) 307*3a48d212SMark Brown goto err; 308f033c26dSMark Brown 309f033c26dSMark Brown return 0; 310f033c26dSMark Brown 311f033c26dSMark Brown err: 312f033c26dSMark Brown regcache_maple_exit(map); 313f033c26dSMark Brown return ret; 314f033c26dSMark Brown } 315f033c26dSMark Brown 316f033c26dSMark Brown struct regcache_ops regcache_maple_ops = { 317f033c26dSMark Brown .type = REGCACHE_MAPLE, 318f033c26dSMark Brown .name = "maple", 319f033c26dSMark Brown .init = regcache_maple_init, 320f033c26dSMark Brown .exit = regcache_maple_exit, 321f033c26dSMark Brown .read = regcache_maple_read, 322f033c26dSMark Brown .write = regcache_maple_write, 323f033c26dSMark Brown .drop = regcache_maple_drop, 324f033c26dSMark Brown .sync = regcache_maple_sync, 325f033c26dSMark Brown }; 326