1 /* 2 * Copyright (C) 2014 Sergey Senozhatsky. 3 * 4 * This program is free software; you can redistribute it and/or 5 * modify it under the terms of the GNU General Public License 6 * as published by the Free Software Foundation; either version 7 * 2 of the License, or (at your option) any later version. 8 */ 9 10 #include <linux/kernel.h> 11 #include <linux/string.h> 12 #include <linux/err.h> 13 #include <linux/slab.h> 14 #include <linux/wait.h> 15 #include <linux/sched.h> 16 17 #include "zcomp.h" 18 #include "zcomp_lzo.h" 19 #ifdef CONFIG_ZRAM_LZ4_COMPRESS 20 #include "zcomp_lz4.h" 21 #endif 22 23 /* 24 * single zcomp_strm backend 25 */ 26 struct zcomp_strm_single { 27 struct mutex strm_lock; 28 struct zcomp_strm *zstrm; 29 }; 30 31 /* 32 * multi zcomp_strm backend 33 */ 34 struct zcomp_strm_multi { 35 /* protect strm list */ 36 spinlock_t strm_lock; 37 /* max possible number of zstrm streams */ 38 int max_strm; 39 /* number of available zstrm streams */ 40 int avail_strm; 41 /* list of available strms */ 42 struct list_head idle_strm; 43 wait_queue_head_t strm_wait; 44 }; 45 46 static struct zcomp_backend *backends[] = { 47 &zcomp_lzo, 48 #ifdef CONFIG_ZRAM_LZ4_COMPRESS 49 &zcomp_lz4, 50 #endif 51 NULL 52 }; 53 54 static struct zcomp_backend *find_backend(const char *compress) 55 { 56 int i = 0; 57 while (backends[i]) { 58 if (sysfs_streq(compress, backends[i]->name)) 59 break; 60 i++; 61 } 62 return backends[i]; 63 } 64 65 static void zcomp_strm_free(struct zcomp *comp, struct zcomp_strm *zstrm) 66 { 67 if (zstrm->private) 68 comp->backend->destroy(zstrm->private); 69 free_pages((unsigned long)zstrm->buffer, 1); 70 kfree(zstrm); 71 } 72 73 /* 74 * allocate new zcomp_strm structure with ->private initialized by 75 * backend, return NULL on error 76 */ 77 static struct zcomp_strm *zcomp_strm_alloc(struct zcomp *comp, gfp_t flags) 78 { 79 struct zcomp_strm *zstrm = kmalloc(sizeof(*zstrm), flags); 80 if (!zstrm) 81 return NULL; 82 83 zstrm->private = comp->backend->create(flags); 84 /* 85 * allocate 2 pages. 1 for compressed data, plus 1 extra for the 86 * case when compressed size is larger than the original one 87 */ 88 zstrm->buffer = (void *)__get_free_pages(flags | __GFP_ZERO, 1); 89 if (!zstrm->private || !zstrm->buffer) { 90 zcomp_strm_free(comp, zstrm); 91 zstrm = NULL; 92 } 93 return zstrm; 94 } 95 96 /* 97 * get idle zcomp_strm or wait until other process release 98 * (zcomp_strm_release()) one for us 99 */ 100 static struct zcomp_strm *zcomp_strm_multi_find(struct zcomp *comp) 101 { 102 struct zcomp_strm_multi *zs = comp->stream; 103 struct zcomp_strm *zstrm; 104 105 while (1) { 106 spin_lock(&zs->strm_lock); 107 if (!list_empty(&zs->idle_strm)) { 108 zstrm = list_entry(zs->idle_strm.next, 109 struct zcomp_strm, list); 110 list_del(&zstrm->list); 111 spin_unlock(&zs->strm_lock); 112 return zstrm; 113 } 114 /* zstrm streams limit reached, wait for idle stream */ 115 if (zs->avail_strm >= zs->max_strm) { 116 spin_unlock(&zs->strm_lock); 117 wait_event(zs->strm_wait, !list_empty(&zs->idle_strm)); 118 continue; 119 } 120 /* allocate new zstrm stream */ 121 zs->avail_strm++; 122 spin_unlock(&zs->strm_lock); 123 /* 124 * This function can be called in swapout/fs write path 125 * so we can't use GFP_FS|IO. And it assumes we already 126 * have at least one stream in zram initialization so we 127 * don't do best effort to allocate more stream in here. 128 * A default stream will work well without further multiple 129 * streams. That's why we use NORETRY | NOWARN. 130 */ 131 zstrm = zcomp_strm_alloc(comp, GFP_NOIO | __GFP_NORETRY | 132 __GFP_NOWARN); 133 if (!zstrm) { 134 spin_lock(&zs->strm_lock); 135 zs->avail_strm--; 136 spin_unlock(&zs->strm_lock); 137 wait_event(zs->strm_wait, !list_empty(&zs->idle_strm)); 138 continue; 139 } 140 break; 141 } 142 return zstrm; 143 } 144 145 /* add stream back to idle list and wake up waiter or free the stream */ 146 static void zcomp_strm_multi_release(struct zcomp *comp, struct zcomp_strm *zstrm) 147 { 148 struct zcomp_strm_multi *zs = comp->stream; 149 150 spin_lock(&zs->strm_lock); 151 if (zs->avail_strm <= zs->max_strm) { 152 list_add(&zstrm->list, &zs->idle_strm); 153 spin_unlock(&zs->strm_lock); 154 wake_up(&zs->strm_wait); 155 return; 156 } 157 158 zs->avail_strm--; 159 spin_unlock(&zs->strm_lock); 160 zcomp_strm_free(comp, zstrm); 161 } 162 163 /* change max_strm limit */ 164 static bool zcomp_strm_multi_set_max_streams(struct zcomp *comp, int num_strm) 165 { 166 struct zcomp_strm_multi *zs = comp->stream; 167 struct zcomp_strm *zstrm; 168 169 spin_lock(&zs->strm_lock); 170 zs->max_strm = num_strm; 171 /* 172 * if user has lowered the limit and there are idle streams, 173 * immediately free as much streams (and memory) as we can. 174 */ 175 while (zs->avail_strm > num_strm && !list_empty(&zs->idle_strm)) { 176 zstrm = list_entry(zs->idle_strm.next, 177 struct zcomp_strm, list); 178 list_del(&zstrm->list); 179 zcomp_strm_free(comp, zstrm); 180 zs->avail_strm--; 181 } 182 spin_unlock(&zs->strm_lock); 183 return true; 184 } 185 186 static void zcomp_strm_multi_destroy(struct zcomp *comp) 187 { 188 struct zcomp_strm_multi *zs = comp->stream; 189 struct zcomp_strm *zstrm; 190 191 while (!list_empty(&zs->idle_strm)) { 192 zstrm = list_entry(zs->idle_strm.next, 193 struct zcomp_strm, list); 194 list_del(&zstrm->list); 195 zcomp_strm_free(comp, zstrm); 196 } 197 kfree(zs); 198 } 199 200 static int zcomp_strm_multi_create(struct zcomp *comp, int max_strm) 201 { 202 struct zcomp_strm *zstrm; 203 struct zcomp_strm_multi *zs; 204 205 comp->destroy = zcomp_strm_multi_destroy; 206 comp->strm_find = zcomp_strm_multi_find; 207 comp->strm_release = zcomp_strm_multi_release; 208 comp->set_max_streams = zcomp_strm_multi_set_max_streams; 209 zs = kmalloc(sizeof(struct zcomp_strm_multi), GFP_KERNEL); 210 if (!zs) 211 return -ENOMEM; 212 213 comp->stream = zs; 214 spin_lock_init(&zs->strm_lock); 215 INIT_LIST_HEAD(&zs->idle_strm); 216 init_waitqueue_head(&zs->strm_wait); 217 zs->max_strm = max_strm; 218 zs->avail_strm = 1; 219 220 zstrm = zcomp_strm_alloc(comp, GFP_KERNEL); 221 if (!zstrm) { 222 kfree(zs); 223 return -ENOMEM; 224 } 225 list_add(&zstrm->list, &zs->idle_strm); 226 return 0; 227 } 228 229 static struct zcomp_strm *zcomp_strm_single_find(struct zcomp *comp) 230 { 231 struct zcomp_strm_single *zs = comp->stream; 232 mutex_lock(&zs->strm_lock); 233 return zs->zstrm; 234 } 235 236 static void zcomp_strm_single_release(struct zcomp *comp, 237 struct zcomp_strm *zstrm) 238 { 239 struct zcomp_strm_single *zs = comp->stream; 240 mutex_unlock(&zs->strm_lock); 241 } 242 243 static bool zcomp_strm_single_set_max_streams(struct zcomp *comp, int num_strm) 244 { 245 /* zcomp_strm_single support only max_comp_streams == 1 */ 246 return false; 247 } 248 249 static void zcomp_strm_single_destroy(struct zcomp *comp) 250 { 251 struct zcomp_strm_single *zs = comp->stream; 252 zcomp_strm_free(comp, zs->zstrm); 253 kfree(zs); 254 } 255 256 static int zcomp_strm_single_create(struct zcomp *comp) 257 { 258 struct zcomp_strm_single *zs; 259 260 comp->destroy = zcomp_strm_single_destroy; 261 comp->strm_find = zcomp_strm_single_find; 262 comp->strm_release = zcomp_strm_single_release; 263 comp->set_max_streams = zcomp_strm_single_set_max_streams; 264 zs = kmalloc(sizeof(struct zcomp_strm_single), GFP_KERNEL); 265 if (!zs) 266 return -ENOMEM; 267 268 comp->stream = zs; 269 mutex_init(&zs->strm_lock); 270 zs->zstrm = zcomp_strm_alloc(comp, GFP_KERNEL); 271 if (!zs->zstrm) { 272 kfree(zs); 273 return -ENOMEM; 274 } 275 return 0; 276 } 277 278 /* show available compressors */ 279 ssize_t zcomp_available_show(const char *comp, char *buf) 280 { 281 ssize_t sz = 0; 282 int i = 0; 283 284 while (backends[i]) { 285 if (!strcmp(comp, backends[i]->name)) 286 sz += scnprintf(buf + sz, PAGE_SIZE - sz - 2, 287 "[%s] ", backends[i]->name); 288 else 289 sz += scnprintf(buf + sz, PAGE_SIZE - sz - 2, 290 "%s ", backends[i]->name); 291 i++; 292 } 293 sz += scnprintf(buf + sz, PAGE_SIZE - sz, "\n"); 294 return sz; 295 } 296 297 bool zcomp_available_algorithm(const char *comp) 298 { 299 return find_backend(comp) != NULL; 300 } 301 302 bool zcomp_set_max_streams(struct zcomp *comp, int num_strm) 303 { 304 return comp->set_max_streams(comp, num_strm); 305 } 306 307 struct zcomp_strm *zcomp_strm_find(struct zcomp *comp) 308 { 309 return comp->strm_find(comp); 310 } 311 312 void zcomp_strm_release(struct zcomp *comp, struct zcomp_strm *zstrm) 313 { 314 comp->strm_release(comp, zstrm); 315 } 316 317 int zcomp_compress(struct zcomp *comp, struct zcomp_strm *zstrm, 318 const unsigned char *src, size_t *dst_len) 319 { 320 return comp->backend->compress(src, zstrm->buffer, dst_len, 321 zstrm->private); 322 } 323 324 int zcomp_decompress(struct zcomp *comp, const unsigned char *src, 325 size_t src_len, unsigned char *dst) 326 { 327 return comp->backend->decompress(src, src_len, dst); 328 } 329 330 void zcomp_destroy(struct zcomp *comp) 331 { 332 comp->destroy(comp); 333 kfree(comp); 334 } 335 336 /* 337 * search available compressors for requested algorithm. 338 * allocate new zcomp and initialize it. return compressing 339 * backend pointer or ERR_PTR if things went bad. ERR_PTR(-EINVAL) 340 * if requested algorithm is not supported, ERR_PTR(-ENOMEM) in 341 * case of allocation error, or any other error potentially 342 * returned by functions zcomp_strm_{multi,single}_create. 343 */ 344 struct zcomp *zcomp_create(const char *compress, int max_strm) 345 { 346 struct zcomp *comp; 347 struct zcomp_backend *backend; 348 int error; 349 350 backend = find_backend(compress); 351 if (!backend) 352 return ERR_PTR(-EINVAL); 353 354 comp = kzalloc(sizeof(struct zcomp), GFP_KERNEL); 355 if (!comp) 356 return ERR_PTR(-ENOMEM); 357 358 comp->backend = backend; 359 if (max_strm > 1) 360 error = zcomp_strm_multi_create(comp, max_strm); 361 else 362 error = zcomp_strm_single_create(comp); 363 if (error) { 364 kfree(comp); 365 return ERR_PTR(error); 366 } 367 return comp; 368 } 369