1f033c26dSMark Brown // SPDX-License-Identifier: GPL-2.0 2f033c26dSMark Brown // 3f033c26dSMark Brown // Register cache access API - maple tree based cache 4f033c26dSMark Brown // 5f033c26dSMark Brown // Copyright 2023 Arm, Ltd 6f033c26dSMark Brown // 7f033c26dSMark Brown // Author: Mark Brown <broonie@kernel.org> 8f033c26dSMark Brown 9f033c26dSMark Brown #include <linux/debugfs.h> 10f033c26dSMark Brown #include <linux/device.h> 11f033c26dSMark Brown #include <linux/maple_tree.h> 12f033c26dSMark Brown #include <linux/slab.h> 13f033c26dSMark Brown 14f033c26dSMark Brown #include "internal.h" 15f033c26dSMark Brown 16f033c26dSMark Brown static int regcache_maple_read(struct regmap *map, 17f033c26dSMark Brown unsigned int reg, unsigned int *value) 18f033c26dSMark Brown { 19f033c26dSMark Brown struct maple_tree *mt = map->cache; 20f033c26dSMark Brown MA_STATE(mas, mt, reg, reg); 21f033c26dSMark Brown unsigned long *entry; 22f033c26dSMark Brown 23f033c26dSMark Brown rcu_read_lock(); 24f033c26dSMark Brown 25fac79badSMark Brown entry = mas_walk(&mas); 26f033c26dSMark Brown if (!entry) { 27f033c26dSMark Brown rcu_read_unlock(); 28f033c26dSMark Brown return -ENOENT; 29f033c26dSMark Brown } 30f033c26dSMark Brown 31f033c26dSMark Brown *value = entry[reg - mas.index]; 32f033c26dSMark Brown 33f033c26dSMark Brown rcu_read_unlock(); 34f033c26dSMark Brown 35f033c26dSMark Brown return 0; 36f033c26dSMark Brown } 37f033c26dSMark Brown 38f033c26dSMark Brown static int regcache_maple_write(struct regmap *map, unsigned int reg, 39f033c26dSMark Brown unsigned int val) 40f033c26dSMark Brown { 41f033c26dSMark Brown struct maple_tree *mt = map->cache; 42f033c26dSMark Brown MA_STATE(mas, mt, reg, reg); 43f033c26dSMark Brown unsigned long *entry, *upper, *lower; 44f033c26dSMark Brown unsigned long index, last; 45f033c26dSMark Brown size_t lower_sz, upper_sz; 46f033c26dSMark Brown int ret; 47f033c26dSMark Brown 48f033c26dSMark Brown rcu_read_lock(); 49f033c26dSMark Brown 50fac79badSMark Brown entry = mas_walk(&mas); 51f033c26dSMark Brown if (entry) { 52f033c26dSMark Brown entry[reg - mas.index] = val; 53f033c26dSMark Brown rcu_read_unlock(); 54f033c26dSMark Brown return 0; 55f033c26dSMark Brown } 56f033c26dSMark Brown 57f033c26dSMark Brown /* Any adjacent entries to extend/merge? */ 58f033c26dSMark Brown mas_set_range(&mas, reg - 1, reg + 1); 59f033c26dSMark Brown index = reg; 60f033c26dSMark Brown last = reg; 61f033c26dSMark Brown 62f033c26dSMark Brown lower = mas_find(&mas, reg - 1); 63f033c26dSMark Brown if (lower) { 64f033c26dSMark Brown index = mas.index; 65f033c26dSMark Brown lower_sz = (mas.last - mas.index + 1) * sizeof(unsigned long); 66f033c26dSMark Brown } 67f033c26dSMark Brown 68f033c26dSMark Brown upper = mas_find(&mas, reg + 1); 69f033c26dSMark Brown if (upper) { 70f033c26dSMark Brown last = mas.last; 71f033c26dSMark Brown upper_sz = (mas.last - mas.index + 1) * sizeof(unsigned long); 72f033c26dSMark Brown } 73f033c26dSMark Brown 74f033c26dSMark Brown rcu_read_unlock(); 75f033c26dSMark Brown 76f033c26dSMark Brown entry = kmalloc((last - index + 1) * sizeof(unsigned long), 77f033c26dSMark Brown GFP_KERNEL); 78f033c26dSMark Brown if (!entry) 79f033c26dSMark Brown return -ENOMEM; 80f033c26dSMark Brown 81f033c26dSMark Brown if (lower) 82f033c26dSMark Brown memcpy(entry, lower, lower_sz); 83f033c26dSMark Brown entry[reg - index] = val; 84f033c26dSMark Brown if (upper) 85f033c26dSMark Brown memcpy(&entry[reg - index + 1], upper, upper_sz); 86f033c26dSMark Brown 87f033c26dSMark Brown /* 88f033c26dSMark Brown * This is safe because the regmap lock means the Maple lock 89f033c26dSMark Brown * is redundant, but we need to take it due to lockdep asserts 90f033c26dSMark Brown * in the maple tree code. 91f033c26dSMark Brown */ 92f033c26dSMark Brown mas_lock(&mas); 93f033c26dSMark Brown 94f033c26dSMark Brown mas_set_range(&mas, index, last); 95f033c26dSMark Brown ret = mas_store_gfp(&mas, entry, GFP_KERNEL); 96f033c26dSMark Brown 97f033c26dSMark Brown mas_unlock(&mas); 98f033c26dSMark Brown 99f033c26dSMark Brown if (ret == 0) { 100f033c26dSMark Brown kfree(lower); 101f033c26dSMark Brown kfree(upper); 102f033c26dSMark Brown } 103f033c26dSMark Brown 104f033c26dSMark Brown return ret; 105f033c26dSMark Brown } 106f033c26dSMark Brown 107f033c26dSMark Brown static int regcache_maple_drop(struct regmap *map, unsigned int min, 108f033c26dSMark Brown unsigned int max) 109f033c26dSMark Brown { 110f033c26dSMark Brown struct maple_tree *mt = map->cache; 111f033c26dSMark Brown MA_STATE(mas, mt, min, max); 112f033c26dSMark Brown unsigned long *entry, *lower, *upper; 113f033c26dSMark Brown unsigned long lower_index, lower_last; 114f033c26dSMark Brown unsigned long upper_index, upper_last; 115f033c26dSMark Brown int ret; 116f033c26dSMark Brown 117f033c26dSMark Brown lower = NULL; 118f033c26dSMark Brown upper = NULL; 119f033c26dSMark Brown 120f033c26dSMark Brown mas_lock(&mas); 121f033c26dSMark Brown 122f033c26dSMark Brown mas_for_each(&mas, entry, max) { 123f033c26dSMark Brown /* 124f033c26dSMark Brown * This is safe because the regmap lock means the 125f033c26dSMark Brown * Maple lock is redundant, but we need to take it due 126f033c26dSMark Brown * to lockdep asserts in the maple tree code. 127f033c26dSMark Brown */ 128f033c26dSMark Brown mas_unlock(&mas); 129f033c26dSMark Brown 130f033c26dSMark Brown /* Do we need to save any of this entry? */ 131f033c26dSMark Brown if (mas.index < min) { 132f033c26dSMark Brown lower_index = mas.index; 133f033c26dSMark Brown lower_last = min -1; 134f033c26dSMark Brown 135f033c26dSMark Brown lower = kmemdup(entry, ((min - mas.index) * 136f033c26dSMark Brown sizeof(unsigned long)), 137f033c26dSMark Brown GFP_KERNEL); 138f033c26dSMark Brown if (!lower) { 139f033c26dSMark Brown ret = -ENOMEM; 140451941acSMark Brown goto out_unlocked; 141f033c26dSMark Brown } 142f033c26dSMark Brown } 143f033c26dSMark Brown 144f033c26dSMark Brown if (mas.last > max) { 145f033c26dSMark Brown upper_index = max + 1; 146f033c26dSMark Brown upper_last = mas.last; 147f033c26dSMark Brown 148f033c26dSMark Brown upper = kmemdup(&entry[max + 1], 149f033c26dSMark Brown ((mas.last - max) * 150f033c26dSMark Brown sizeof(unsigned long)), 151f033c26dSMark Brown GFP_KERNEL); 152f033c26dSMark Brown if (!upper) { 153f033c26dSMark Brown ret = -ENOMEM; 154451941acSMark Brown goto out_unlocked; 155f033c26dSMark Brown } 156f033c26dSMark Brown } 157f033c26dSMark Brown 158f033c26dSMark Brown kfree(entry); 159f033c26dSMark Brown mas_lock(&mas); 160f033c26dSMark Brown mas_erase(&mas); 161f033c26dSMark Brown 162f033c26dSMark Brown /* Insert new nodes with the saved data */ 163f033c26dSMark Brown if (lower) { 164f033c26dSMark Brown mas_set_range(&mas, lower_index, lower_last); 165f033c26dSMark Brown ret = mas_store_gfp(&mas, lower, GFP_KERNEL); 166f033c26dSMark Brown if (ret != 0) 167f033c26dSMark Brown goto out; 168f033c26dSMark Brown lower = NULL; 169f033c26dSMark Brown } 170f033c26dSMark Brown 171f033c26dSMark Brown if (upper) { 172f033c26dSMark Brown mas_set_range(&mas, upper_index, upper_last); 173f033c26dSMark Brown ret = mas_store_gfp(&mas, upper, GFP_KERNEL); 174f033c26dSMark Brown if (ret != 0) 175f033c26dSMark Brown goto out; 176f033c26dSMark Brown upper = NULL; 177f033c26dSMark Brown } 178f033c26dSMark Brown } 179f033c26dSMark Brown 180f033c26dSMark Brown out: 181f033c26dSMark Brown mas_unlock(&mas); 182451941acSMark Brown out_unlocked: 183f033c26dSMark Brown kfree(lower); 184f033c26dSMark Brown kfree(upper); 185f033c26dSMark Brown 186f033c26dSMark Brown return ret; 187f033c26dSMark Brown } 188f033c26dSMark Brown 189*bfa0b38cSMark Brown static int regcache_maple_sync_block(struct regmap *map, unsigned long *entry, 190*bfa0b38cSMark Brown struct ma_state *mas, 191*bfa0b38cSMark Brown unsigned int min, unsigned int max) 192*bfa0b38cSMark Brown { 193*bfa0b38cSMark Brown void *buf; 194*bfa0b38cSMark Brown unsigned long r; 195*bfa0b38cSMark Brown size_t val_bytes = map->format.val_bytes; 196*bfa0b38cSMark Brown int ret = 0; 197*bfa0b38cSMark Brown 198*bfa0b38cSMark Brown mas_pause(mas); 199*bfa0b38cSMark Brown rcu_read_unlock(); 200*bfa0b38cSMark Brown 201*bfa0b38cSMark Brown /* 202*bfa0b38cSMark Brown * Use a raw write if writing more than one register to a 203*bfa0b38cSMark Brown * device that supports raw writes to reduce transaction 204*bfa0b38cSMark Brown * overheads. 205*bfa0b38cSMark Brown */ 206*bfa0b38cSMark Brown if (max - min > 1 && regmap_can_raw_write(map)) { 207*bfa0b38cSMark Brown buf = kmalloc(val_bytes * (max - min), map->alloc_flags); 208*bfa0b38cSMark Brown if (!buf) { 209*bfa0b38cSMark Brown ret = -ENOMEM; 210*bfa0b38cSMark Brown goto out; 211*bfa0b38cSMark Brown } 212*bfa0b38cSMark Brown 213*bfa0b38cSMark Brown /* Render the data for a raw write */ 214*bfa0b38cSMark Brown for (r = min; r < max; r++) { 215*bfa0b38cSMark Brown regcache_set_val(map, buf, r - min, 216*bfa0b38cSMark Brown entry[r - mas->index]); 217*bfa0b38cSMark Brown } 218*bfa0b38cSMark Brown 219*bfa0b38cSMark Brown ret = _regmap_raw_write(map, min, buf, (max - min) * val_bytes, 220*bfa0b38cSMark Brown false); 221*bfa0b38cSMark Brown 222*bfa0b38cSMark Brown kfree(buf); 223*bfa0b38cSMark Brown } else { 224*bfa0b38cSMark Brown for (r = min; r < max; r++) { 225*bfa0b38cSMark Brown ret = _regmap_write(map, r, 226*bfa0b38cSMark Brown entry[r - mas->index]); 227*bfa0b38cSMark Brown if (ret != 0) 228*bfa0b38cSMark Brown goto out; 229*bfa0b38cSMark Brown } 230*bfa0b38cSMark Brown } 231*bfa0b38cSMark Brown 232*bfa0b38cSMark Brown out: 233*bfa0b38cSMark Brown rcu_read_lock(); 234*bfa0b38cSMark Brown 235*bfa0b38cSMark Brown return ret; 236*bfa0b38cSMark Brown } 237*bfa0b38cSMark Brown 238f033c26dSMark Brown static int regcache_maple_sync(struct regmap *map, unsigned int min, 239f033c26dSMark Brown unsigned int max) 240f033c26dSMark Brown { 241f033c26dSMark Brown struct maple_tree *mt = map->cache; 242f033c26dSMark Brown unsigned long *entry; 243f033c26dSMark Brown MA_STATE(mas, mt, min, max); 244f033c26dSMark Brown unsigned long lmin = min; 245f033c26dSMark Brown unsigned long lmax = max; 246*bfa0b38cSMark Brown unsigned int r, v, sync_start; 247f033c26dSMark Brown int ret; 248*bfa0b38cSMark Brown bool sync_needed = false; 249f033c26dSMark Brown 250f033c26dSMark Brown map->cache_bypass = true; 251f033c26dSMark Brown 252f033c26dSMark Brown rcu_read_lock(); 253f033c26dSMark Brown 254f033c26dSMark Brown mas_for_each(&mas, entry, max) { 255f033c26dSMark Brown for (r = max(mas.index, lmin); r <= min(mas.last, lmax); r++) { 256*bfa0b38cSMark Brown v = entry[r - mas.index]; 257*bfa0b38cSMark Brown 258*bfa0b38cSMark Brown if (regcache_reg_needs_sync(map, r, v)) { 259*bfa0b38cSMark Brown if (!sync_needed) { 260*bfa0b38cSMark Brown sync_start = r; 261*bfa0b38cSMark Brown sync_needed = true; 262*bfa0b38cSMark Brown } 263*bfa0b38cSMark Brown continue; 264*bfa0b38cSMark Brown } 265*bfa0b38cSMark Brown 266*bfa0b38cSMark Brown if (!sync_needed) 267*bfa0b38cSMark Brown continue; 268*bfa0b38cSMark Brown 269*bfa0b38cSMark Brown ret = regcache_maple_sync_block(map, entry, &mas, 270*bfa0b38cSMark Brown sync_start, r); 271f033c26dSMark Brown if (ret != 0) 272f033c26dSMark Brown goto out; 273*bfa0b38cSMark Brown sync_needed = false; 274f033c26dSMark Brown } 275f033c26dSMark Brown 276*bfa0b38cSMark Brown if (sync_needed) { 277*bfa0b38cSMark Brown ret = regcache_maple_sync_block(map, entry, &mas, 278*bfa0b38cSMark Brown sync_start, r); 279*bfa0b38cSMark Brown if (ret != 0) 280*bfa0b38cSMark Brown goto out; 281*bfa0b38cSMark Brown sync_needed = false; 282*bfa0b38cSMark Brown } 283*bfa0b38cSMark Brown } 284f033c26dSMark Brown 2850cc65780SMark Brown out: 286*bfa0b38cSMark Brown rcu_read_unlock(); 287*bfa0b38cSMark Brown 288f033c26dSMark Brown map->cache_bypass = false; 289f033c26dSMark Brown 290f033c26dSMark Brown return ret; 291f033c26dSMark Brown } 292f033c26dSMark Brown 293f033c26dSMark Brown static int regcache_maple_exit(struct regmap *map) 294f033c26dSMark Brown { 295f033c26dSMark Brown struct maple_tree *mt = map->cache; 296f033c26dSMark Brown MA_STATE(mas, mt, 0, UINT_MAX); 297f033c26dSMark Brown unsigned int *entry;; 298f033c26dSMark Brown 299f033c26dSMark Brown /* if we've already been called then just return */ 300f033c26dSMark Brown if (!mt) 301f033c26dSMark Brown return 0; 302f033c26dSMark Brown 303f033c26dSMark Brown mas_lock(&mas); 304f033c26dSMark Brown mas_for_each(&mas, entry, UINT_MAX) 305f033c26dSMark Brown kfree(entry); 306f033c26dSMark Brown __mt_destroy(mt); 307f033c26dSMark Brown mas_unlock(&mas); 308f033c26dSMark Brown 309f033c26dSMark Brown kfree(mt); 310f033c26dSMark Brown map->cache = NULL; 311f033c26dSMark Brown 312f033c26dSMark Brown return 0; 313f033c26dSMark Brown } 314f033c26dSMark Brown 3153a48d212SMark Brown static int regcache_maple_insert_block(struct regmap *map, int first, 3163a48d212SMark Brown int last) 3173a48d212SMark Brown { 3183a48d212SMark Brown struct maple_tree *mt = map->cache; 3193a48d212SMark Brown MA_STATE(mas, mt, first, last); 3203a48d212SMark Brown unsigned long *entry; 3213a48d212SMark Brown int i, ret; 3223a48d212SMark Brown 3233a48d212SMark Brown entry = kcalloc(last - first + 1, sizeof(unsigned long), GFP_KERNEL); 3243a48d212SMark Brown if (!entry) 3253a48d212SMark Brown return -ENOMEM; 3263a48d212SMark Brown 3273a48d212SMark Brown for (i = 0; i < last - first + 1; i++) 3283a48d212SMark Brown entry[i] = map->reg_defaults[first + i].def; 3293a48d212SMark Brown 3303a48d212SMark Brown mas_lock(&mas); 3313a48d212SMark Brown 3323a48d212SMark Brown mas_set_range(&mas, map->reg_defaults[first].reg, 3333a48d212SMark Brown map->reg_defaults[last].reg); 3343a48d212SMark Brown ret = mas_store_gfp(&mas, entry, GFP_KERNEL); 3353a48d212SMark Brown 3363a48d212SMark Brown mas_unlock(&mas); 3373a48d212SMark Brown 3383a48d212SMark Brown if (ret) 3393a48d212SMark Brown kfree(entry); 3403a48d212SMark Brown 3413a48d212SMark Brown return ret; 3423a48d212SMark Brown } 3433a48d212SMark Brown 344f033c26dSMark Brown static int regcache_maple_init(struct regmap *map) 345f033c26dSMark Brown { 346f033c26dSMark Brown struct maple_tree *mt; 347f033c26dSMark Brown int i; 348f033c26dSMark Brown int ret; 3493a48d212SMark Brown int range_start; 350f033c26dSMark Brown 351f033c26dSMark Brown mt = kmalloc(sizeof(*mt), GFP_KERNEL); 352f033c26dSMark Brown if (!mt) 353f033c26dSMark Brown return -ENOMEM; 354f033c26dSMark Brown map->cache = mt; 355f033c26dSMark Brown 356f033c26dSMark Brown mt_init(mt); 357f033c26dSMark Brown 3583a48d212SMark Brown if (!map->num_reg_defaults) 3593a48d212SMark Brown return 0; 3603a48d212SMark Brown 3613a48d212SMark Brown range_start = 0; 3623a48d212SMark Brown 3633a48d212SMark Brown /* Scan for ranges of contiguous registers */ 3643a48d212SMark Brown for (i = 1; i < map->num_reg_defaults; i++) { 3653a48d212SMark Brown if (map->reg_defaults[i].reg != 3663a48d212SMark Brown map->reg_defaults[i - 1].reg + 1) { 3673a48d212SMark Brown ret = regcache_maple_insert_block(map, range_start, 3683a48d212SMark Brown i - 1); 3693a48d212SMark Brown if (ret != 0) 370f033c26dSMark Brown goto err; 3713a48d212SMark Brown 3723a48d212SMark Brown range_start = i; 373f033c26dSMark Brown } 3743a48d212SMark Brown } 3753a48d212SMark Brown 3763a48d212SMark Brown /* Add the last block */ 3773a48d212SMark Brown ret = regcache_maple_insert_block(map, range_start, 3783a48d212SMark Brown map->num_reg_defaults - 1); 3793a48d212SMark Brown if (ret != 0) 3803a48d212SMark Brown goto err; 381f033c26dSMark Brown 382f033c26dSMark Brown return 0; 383f033c26dSMark Brown 384f033c26dSMark Brown err: 385f033c26dSMark Brown regcache_maple_exit(map); 386f033c26dSMark Brown return ret; 387f033c26dSMark Brown } 388f033c26dSMark Brown 389f033c26dSMark Brown struct regcache_ops regcache_maple_ops = { 390f033c26dSMark Brown .type = REGCACHE_MAPLE, 391f033c26dSMark Brown .name = "maple", 392f033c26dSMark Brown .init = regcache_maple_init, 393f033c26dSMark Brown .exit = regcache_maple_exit, 394f033c26dSMark Brown .read = regcache_maple_read, 395f033c26dSMark Brown .write = regcache_maple_write, 396f033c26dSMark Brown .drop = regcache_maple_drop, 397f033c26dSMark Brown .sync = regcache_maple_sync, 398f033c26dSMark Brown }; 399