1 /* 2 * Copyright (C) 2014 Sergey Senozhatsky. 3 * 4 * This program is free software; you can redistribute it and/or 5 * modify it under the terms of the GNU General Public License 6 * as published by the Free Software Foundation; either version 7 * 2 of the License, or (at your option) any later version. 8 */ 9 10 #include <linux/kernel.h> 11 #include <linux/string.h> 12 #include <linux/err.h> 13 #include <linux/slab.h> 14 #include <linux/wait.h> 15 #include <linux/sched.h> 16 #include <linux/cpu.h> 17 #include <linux/crypto.h> 18 19 #include "zcomp.h" 20 21 static const char * const backends[] = { 22 "lzo", 23 #if IS_ENABLED(CONFIG_CRYPTO_LZ4) 24 "lz4", 25 #endif 26 #if IS_ENABLED(CONFIG_CRYPTO_DEFLATE) 27 "deflate", 28 #endif 29 #if IS_ENABLED(CONFIG_CRYPTO_LZ4HC) 30 "lz4hc", 31 #endif 32 #if IS_ENABLED(CONFIG_CRYPTO_842) 33 "842", 34 #endif 35 NULL 36 }; 37 38 static void zcomp_strm_free(struct zcomp_strm *zstrm) 39 { 40 if (!IS_ERR_OR_NULL(zstrm->tfm)) 41 crypto_free_comp(zstrm->tfm); 42 free_pages((unsigned long)zstrm->buffer, 1); 43 kfree(zstrm); 44 } 45 46 /* 47 * allocate new zcomp_strm structure with ->tfm initialized by 48 * backend, return NULL on error 49 */ 50 static struct zcomp_strm *zcomp_strm_alloc(struct zcomp *comp) 51 { 52 struct zcomp_strm *zstrm = kmalloc(sizeof(*zstrm), GFP_KERNEL); 53 if (!zstrm) 54 return NULL; 55 56 zstrm->tfm = crypto_alloc_comp(comp->name, 0, 0); 57 /* 58 * allocate 2 pages. 1 for compressed data, plus 1 extra for the 59 * case when compressed size is larger than the original one 60 */ 61 zstrm->buffer = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, 1); 62 if (IS_ERR_OR_NULL(zstrm->tfm) || !zstrm->buffer) { 63 zcomp_strm_free(zstrm); 64 zstrm = NULL; 65 } 66 return zstrm; 67 } 68 69 bool zcomp_available_algorithm(const char *comp) 70 { 71 int i = 0; 72 73 while (backends[i]) { 74 if (sysfs_streq(comp, backends[i])) 75 return true; 76 i++; 77 } 78 79 /* 80 * Crypto does not ignore a trailing new line symbol, 81 * so make sure you don't supply a string containing 82 * one. 83 * This also means that we permit zcomp initialisation 84 * with any compressing algorithm known to crypto api. 85 */ 86 return crypto_has_comp(comp, 0, 0) == 1; 87 } 88 89 /* show available compressors */ 90 ssize_t zcomp_available_show(const char *comp, char *buf) 91 { 92 bool known_algorithm = false; 93 ssize_t sz = 0; 94 int i = 0; 95 96 for (; backends[i]; i++) { 97 if (!strcmp(comp, backends[i])) { 98 known_algorithm = true; 99 sz += scnprintf(buf + sz, PAGE_SIZE - sz - 2, 100 "[%s] ", backends[i]); 101 } else { 102 sz += scnprintf(buf + sz, PAGE_SIZE - sz - 2, 103 "%s ", backends[i]); 104 } 105 } 106 107 /* 108 * Out-of-tree module known to crypto api or a missing 109 * entry in `backends'. 110 */ 111 if (!known_algorithm && crypto_has_comp(comp, 0, 0) == 1) 112 sz += scnprintf(buf + sz, PAGE_SIZE - sz - 2, 113 "[%s] ", comp); 114 115 sz += scnprintf(buf + sz, PAGE_SIZE - sz, "\n"); 116 return sz; 117 } 118 119 struct zcomp_strm *zcomp_stream_get(struct zcomp *comp) 120 { 121 return *get_cpu_ptr(comp->stream); 122 } 123 124 void zcomp_stream_put(struct zcomp *comp) 125 { 126 put_cpu_ptr(comp->stream); 127 } 128 129 int zcomp_compress(struct zcomp_strm *zstrm, 130 const void *src, unsigned int *dst_len) 131 { 132 /* 133 * Our dst memory (zstrm->buffer) is always `2 * PAGE_SIZE' sized 134 * because sometimes we can endup having a bigger compressed data 135 * due to various reasons: for example compression algorithms tend 136 * to add some padding to the compressed buffer. Speaking of padding, 137 * comp algorithm `842' pads the compressed length to multiple of 8 138 * and returns -ENOSP when the dst memory is not big enough, which 139 * is not something that ZRAM wants to see. We can handle the 140 * `compressed_size > PAGE_SIZE' case easily in ZRAM, but when we 141 * receive -ERRNO from the compressing backend we can't help it 142 * anymore. To make `842' happy we need to tell the exact size of 143 * the dst buffer, zram_drv will take care of the fact that 144 * compressed buffer is too big. 145 */ 146 *dst_len = PAGE_SIZE * 2; 147 148 return crypto_comp_compress(zstrm->tfm, 149 src, PAGE_SIZE, 150 zstrm->buffer, dst_len); 151 } 152 153 int zcomp_decompress(struct zcomp_strm *zstrm, 154 const void *src, unsigned int src_len, void *dst) 155 { 156 unsigned int dst_len = PAGE_SIZE; 157 158 return crypto_comp_decompress(zstrm->tfm, 159 src, src_len, 160 dst, &dst_len); 161 } 162 163 static int __zcomp_cpu_notifier(struct zcomp *comp, 164 unsigned long action, unsigned long cpu) 165 { 166 struct zcomp_strm *zstrm; 167 168 switch (action) { 169 case CPU_UP_PREPARE: 170 if (WARN_ON(*per_cpu_ptr(comp->stream, cpu))) 171 break; 172 zstrm = zcomp_strm_alloc(comp); 173 if (IS_ERR_OR_NULL(zstrm)) { 174 pr_err("Can't allocate a compression stream\n"); 175 return NOTIFY_BAD; 176 } 177 *per_cpu_ptr(comp->stream, cpu) = zstrm; 178 break; 179 case CPU_DEAD: 180 case CPU_UP_CANCELED: 181 zstrm = *per_cpu_ptr(comp->stream, cpu); 182 if (!IS_ERR_OR_NULL(zstrm)) 183 zcomp_strm_free(zstrm); 184 *per_cpu_ptr(comp->stream, cpu) = NULL; 185 break; 186 default: 187 break; 188 } 189 return NOTIFY_OK; 190 } 191 192 static int zcomp_cpu_notifier(struct notifier_block *nb, 193 unsigned long action, void *pcpu) 194 { 195 unsigned long cpu = (unsigned long)pcpu; 196 struct zcomp *comp = container_of(nb, typeof(*comp), notifier); 197 198 return __zcomp_cpu_notifier(comp, action, cpu); 199 } 200 201 static int zcomp_init(struct zcomp *comp) 202 { 203 unsigned long cpu; 204 int ret; 205 206 comp->notifier.notifier_call = zcomp_cpu_notifier; 207 208 comp->stream = alloc_percpu(struct zcomp_strm *); 209 if (!comp->stream) 210 return -ENOMEM; 211 212 cpu_notifier_register_begin(); 213 for_each_online_cpu(cpu) { 214 ret = __zcomp_cpu_notifier(comp, CPU_UP_PREPARE, cpu); 215 if (ret == NOTIFY_BAD) 216 goto cleanup; 217 } 218 __register_cpu_notifier(&comp->notifier); 219 cpu_notifier_register_done(); 220 return 0; 221 222 cleanup: 223 for_each_online_cpu(cpu) 224 __zcomp_cpu_notifier(comp, CPU_UP_CANCELED, cpu); 225 cpu_notifier_register_done(); 226 return -ENOMEM; 227 } 228 229 void zcomp_destroy(struct zcomp *comp) 230 { 231 unsigned long cpu; 232 233 cpu_notifier_register_begin(); 234 for_each_online_cpu(cpu) 235 __zcomp_cpu_notifier(comp, CPU_UP_CANCELED, cpu); 236 __unregister_cpu_notifier(&comp->notifier); 237 cpu_notifier_register_done(); 238 239 free_percpu(comp->stream); 240 kfree(comp); 241 } 242 243 /* 244 * search available compressors for requested algorithm. 245 * allocate new zcomp and initialize it. return compressing 246 * backend pointer or ERR_PTR if things went bad. ERR_PTR(-EINVAL) 247 * if requested algorithm is not supported, ERR_PTR(-ENOMEM) in 248 * case of allocation error, or any other error potentially 249 * returned by zcomp_init(). 250 */ 251 struct zcomp *zcomp_create(const char *compress) 252 { 253 struct zcomp *comp; 254 int error; 255 256 if (!zcomp_available_algorithm(compress)) 257 return ERR_PTR(-EINVAL); 258 259 comp = kzalloc(sizeof(struct zcomp), GFP_KERNEL); 260 if (!comp) 261 return ERR_PTR(-ENOMEM); 262 263 comp->name = compress; 264 error = zcomp_init(comp); 265 if (error) { 266 kfree(comp); 267 return ERR_PTR(error); 268 } 269 return comp; 270 } 271