1 /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 /* Internal definitions for FS-Cache 3 * 4 * Copyright (C) 2004-2007 Red Hat, Inc. All Rights Reserved. 5 * Written by David Howells (dhowells@redhat.com) 6 */ 7 8 /* 9 * Lock order, in the order in which multiple locks should be obtained: 10 * - fscache_addremove_sem 11 * - cookie->lock 12 * - cookie->parent->lock 13 * - cache->object_list_lock 14 * - object->lock 15 * - object->parent->lock 16 * - cookie->stores_lock 17 * - fscache_thread_lock 18 * 19 */ 20 21 #ifdef pr_fmt 22 #undef pr_fmt 23 #endif 24 25 #define pr_fmt(fmt) "FS-Cache: " fmt 26 27 #include <linux/fscache-cache.h> 28 #include <trace/events/fscache.h> 29 #include <linux/sched.h> 30 #include <linux/seq_file.h> 31 32 #define FSCACHE_MIN_THREADS 4 33 #define FSCACHE_MAX_THREADS 32 34 35 /* 36 * cache.c 37 */ 38 extern struct list_head fscache_cache_list; 39 extern struct rw_semaphore fscache_addremove_sem; 40 41 extern struct fscache_cache *fscache_select_cache_for_object( 42 struct fscache_cookie *); 43 44 /* 45 * cookie.c 46 */ 47 extern struct kmem_cache *fscache_cookie_jar; 48 extern const struct seq_operations fscache_cookies_seq_ops; 49 50 extern void fscache_free_cookie(struct fscache_cookie *); 51 extern struct fscache_cookie *fscache_alloc_cookie(struct fscache_cookie *, 52 const struct fscache_cookie_def *, 53 const void *, size_t, 54 const void *, size_t, 55 void *, loff_t); 56 extern struct fscache_cookie *fscache_hash_cookie(struct fscache_cookie *); 57 extern struct fscache_cookie *fscache_cookie_get(struct fscache_cookie *, 58 enum fscache_cookie_trace); 59 extern void fscache_cookie_put(struct fscache_cookie *, 60 enum fscache_cookie_trace); 61 62 static inline void fscache_cookie_see(struct fscache_cookie *cookie, 63 enum fscache_cookie_trace where) 64 { 65 trace_fscache_cookie(cookie->debug_id, refcount_read(&cookie->ref), 66 where); 67 } 68 69 /* 70 * fsdef.c 71 */ 72 extern struct fscache_cookie fscache_fsdef_index; 73 extern struct fscache_cookie_def fscache_fsdef_netfs_def; 74 75 /* 76 * main.c 77 */ 78 extern unsigned fscache_defer_lookup; 79 extern unsigned fscache_defer_create; 80 extern unsigned fscache_debug; 81 extern struct kobject *fscache_root; 82 extern struct workqueue_struct *fscache_object_wq; 83 extern struct workqueue_struct *fscache_op_wq; 84 DECLARE_PER_CPU(wait_queue_head_t, fscache_object_cong_wait); 85 86 extern unsigned int fscache_hash(unsigned int salt, unsigned int *data, unsigned int n); 87 88 static inline bool fscache_object_congested(void) 89 { 90 return workqueue_congested(WORK_CPU_UNBOUND, fscache_object_wq); 91 } 92 93 /* 94 * object.c 95 */ 96 extern void fscache_enqueue_object(struct fscache_object *); 97 98 /* 99 * operation.c 100 */ 101 extern int fscache_submit_exclusive_op(struct fscache_object *, 102 struct fscache_operation *); 103 extern int fscache_submit_op(struct fscache_object *, 104 struct fscache_operation *); 105 extern int fscache_cancel_op(struct fscache_operation *, bool); 106 extern void fscache_cancel_all_ops(struct fscache_object *); 107 extern void fscache_abort_object(struct fscache_object *); 108 extern void fscache_start_operations(struct fscache_object *); 109 extern void fscache_operation_gc(struct work_struct *); 110 111 /* 112 * page.c 113 */ 114 extern int fscache_wait_for_deferred_lookup(struct fscache_cookie *); 115 extern int fscache_wait_for_operation_activation(struct fscache_object *, 116 struct fscache_operation *, 117 atomic_t *, 118 atomic_t *); 119 extern void fscache_invalidate_writes(struct fscache_cookie *); 120 struct fscache_retrieval *fscache_alloc_retrieval(struct fscache_cookie *cookie, 121 struct address_space *mapping, 122 fscache_rw_complete_t end_io_func, 123 void *context); 124 125 /* 126 * proc.c 127 */ 128 #ifdef CONFIG_PROC_FS 129 extern int __init fscache_proc_init(void); 130 extern void fscache_proc_cleanup(void); 131 #else 132 #define fscache_proc_init() (0) 133 #define fscache_proc_cleanup() do {} while (0) 134 #endif 135 136 /* 137 * stats.c 138 */ 139 #ifdef CONFIG_FSCACHE_STATS 140 extern atomic_t fscache_n_ops_processed[FSCACHE_MAX_THREADS]; 141 extern atomic_t fscache_n_objs_processed[FSCACHE_MAX_THREADS]; 142 143 extern atomic_t fscache_n_op_pend; 144 extern atomic_t fscache_n_op_run; 145 extern atomic_t fscache_n_op_enqueue; 146 extern atomic_t fscache_n_op_deferred_release; 147 extern atomic_t fscache_n_op_initialised; 148 extern atomic_t fscache_n_op_release; 149 extern atomic_t fscache_n_op_gc; 150 extern atomic_t fscache_n_op_cancelled; 151 extern atomic_t fscache_n_op_rejected; 152 153 extern atomic_t fscache_n_attr_changed; 154 extern atomic_t fscache_n_attr_changed_ok; 155 extern atomic_t fscache_n_attr_changed_nobufs; 156 extern atomic_t fscache_n_attr_changed_nomem; 157 extern atomic_t fscache_n_attr_changed_calls; 158 159 extern atomic_t fscache_n_allocs; 160 extern atomic_t fscache_n_allocs_ok; 161 extern atomic_t fscache_n_allocs_wait; 162 extern atomic_t fscache_n_allocs_nobufs; 163 extern atomic_t fscache_n_allocs_intr; 164 extern atomic_t fscache_n_allocs_object_dead; 165 extern atomic_t fscache_n_alloc_ops; 166 extern atomic_t fscache_n_alloc_op_waits; 167 168 extern atomic_t fscache_n_retrievals; 169 extern atomic_t fscache_n_retrievals_ok; 170 extern atomic_t fscache_n_retrievals_wait; 171 extern atomic_t fscache_n_retrievals_nodata; 172 extern atomic_t fscache_n_retrievals_nobufs; 173 extern atomic_t fscache_n_retrievals_intr; 174 extern atomic_t fscache_n_retrievals_nomem; 175 extern atomic_t fscache_n_retrievals_object_dead; 176 extern atomic_t fscache_n_retrieval_ops; 177 extern atomic_t fscache_n_retrieval_op_waits; 178 179 extern atomic_t fscache_n_stores; 180 extern atomic_t fscache_n_stores_ok; 181 extern atomic_t fscache_n_stores_again; 182 extern atomic_t fscache_n_stores_nobufs; 183 extern atomic_t fscache_n_stores_oom; 184 extern atomic_t fscache_n_store_ops; 185 extern atomic_t fscache_n_store_calls; 186 extern atomic_t fscache_n_store_pages; 187 extern atomic_t fscache_n_store_radix_deletes; 188 extern atomic_t fscache_n_store_pages_over_limit; 189 190 extern atomic_t fscache_n_store_vmscan_not_storing; 191 extern atomic_t fscache_n_store_vmscan_gone; 192 extern atomic_t fscache_n_store_vmscan_busy; 193 extern atomic_t fscache_n_store_vmscan_cancelled; 194 extern atomic_t fscache_n_store_vmscan_wait; 195 196 extern atomic_t fscache_n_marks; 197 extern atomic_t fscache_n_uncaches; 198 199 extern atomic_t fscache_n_acquires; 200 extern atomic_t fscache_n_acquires_null; 201 extern atomic_t fscache_n_acquires_no_cache; 202 extern atomic_t fscache_n_acquires_ok; 203 extern atomic_t fscache_n_acquires_nobufs; 204 extern atomic_t fscache_n_acquires_oom; 205 206 extern atomic_t fscache_n_invalidates; 207 extern atomic_t fscache_n_invalidates_run; 208 209 extern atomic_t fscache_n_updates; 210 extern atomic_t fscache_n_updates_null; 211 extern atomic_t fscache_n_updates_run; 212 213 extern atomic_t fscache_n_relinquishes; 214 extern atomic_t fscache_n_relinquishes_null; 215 extern atomic_t fscache_n_relinquishes_waitcrt; 216 extern atomic_t fscache_n_relinquishes_retire; 217 218 extern atomic_t fscache_n_cookie_index; 219 extern atomic_t fscache_n_cookie_data; 220 extern atomic_t fscache_n_cookie_special; 221 222 extern atomic_t fscache_n_object_alloc; 223 extern atomic_t fscache_n_object_no_alloc; 224 extern atomic_t fscache_n_object_lookups; 225 extern atomic_t fscache_n_object_lookups_negative; 226 extern atomic_t fscache_n_object_lookups_positive; 227 extern atomic_t fscache_n_object_lookups_timed_out; 228 extern atomic_t fscache_n_object_created; 229 extern atomic_t fscache_n_object_avail; 230 extern atomic_t fscache_n_object_dead; 231 232 extern atomic_t fscache_n_checkaux_none; 233 extern atomic_t fscache_n_checkaux_okay; 234 extern atomic_t fscache_n_checkaux_update; 235 extern atomic_t fscache_n_checkaux_obsolete; 236 237 extern atomic_t fscache_n_cop_alloc_object; 238 extern atomic_t fscache_n_cop_lookup_object; 239 extern atomic_t fscache_n_cop_lookup_complete; 240 extern atomic_t fscache_n_cop_grab_object; 241 extern atomic_t fscache_n_cop_invalidate_object; 242 extern atomic_t fscache_n_cop_update_object; 243 extern atomic_t fscache_n_cop_drop_object; 244 extern atomic_t fscache_n_cop_put_object; 245 extern atomic_t fscache_n_cop_sync_cache; 246 extern atomic_t fscache_n_cop_attr_changed; 247 extern atomic_t fscache_n_cop_read_or_alloc_page; 248 extern atomic_t fscache_n_cop_read_or_alloc_pages; 249 extern atomic_t fscache_n_cop_allocate_page; 250 extern atomic_t fscache_n_cop_allocate_pages; 251 extern atomic_t fscache_n_cop_write_page; 252 extern atomic_t fscache_n_cop_uncache_page; 253 extern atomic_t fscache_n_cop_dissociate_pages; 254 255 extern atomic_t fscache_n_cache_no_space_reject; 256 extern atomic_t fscache_n_cache_stale_objects; 257 extern atomic_t fscache_n_cache_retired_objects; 258 extern atomic_t fscache_n_cache_culled_objects; 259 260 static inline void fscache_stat(atomic_t *stat) 261 { 262 atomic_inc(stat); 263 } 264 265 static inline void fscache_stat_d(atomic_t *stat) 266 { 267 atomic_dec(stat); 268 } 269 270 #define __fscache_stat(stat) (stat) 271 272 int fscache_stats_show(struct seq_file *m, void *v); 273 #else 274 275 #define __fscache_stat(stat) (NULL) 276 #define fscache_stat(stat) do {} while (0) 277 #define fscache_stat_d(stat) do {} while (0) 278 #endif 279 280 /* 281 * raise an event on an object 282 * - if the event is not masked for that object, then the object is 283 * queued for attention by the thread pool. 284 */ 285 static inline void fscache_raise_event(struct fscache_object *object, 286 unsigned event) 287 { 288 BUG_ON(event >= NR_FSCACHE_OBJECT_EVENTS); 289 #if 0 290 printk("*** fscache_raise_event(OBJ%d{%lx},%x)\n", 291 object->debug_id, object->event_mask, (1 << event)); 292 #endif 293 if (!test_and_set_bit(event, &object->events) && 294 test_bit(event, &object->event_mask)) 295 fscache_enqueue_object(object); 296 } 297 298 /* 299 * get an extra reference to a netfs retrieval context 300 */ 301 static inline 302 void *fscache_get_context(struct fscache_cookie *cookie, void *context) 303 { 304 if (cookie->def->get_context) 305 cookie->def->get_context(cookie->netfs_data, context); 306 return context; 307 } 308 309 /* 310 * release a reference to a netfs retrieval context 311 */ 312 static inline 313 void fscache_put_context(struct fscache_cookie *cookie, void *context) 314 { 315 if (cookie->def->put_context) 316 cookie->def->put_context(cookie->netfs_data, context); 317 } 318 319 /* 320 * Update the auxiliary data on a cookie. 321 */ 322 static inline 323 void fscache_update_aux(struct fscache_cookie *cookie, const void *aux_data) 324 { 325 void *p; 326 327 if (!aux_data) 328 return; 329 if (cookie->aux_len <= sizeof(cookie->inline_aux)) 330 p = cookie->inline_aux; 331 else 332 p = cookie->aux; 333 334 if (memcmp(p, aux_data, cookie->aux_len) != 0) { 335 memcpy(p, aux_data, cookie->aux_len); 336 set_bit(FSCACHE_COOKIE_AUX_UPDATED, &cookie->flags); 337 } 338 } 339 340 /*****************************************************************************/ 341 /* 342 * debug tracing 343 */ 344 #define dbgprintk(FMT, ...) \ 345 printk(KERN_DEBUG "[%-6.6s] "FMT"\n", current->comm, ##__VA_ARGS__) 346 347 #define kenter(FMT, ...) dbgprintk("==> %s("FMT")", __func__, ##__VA_ARGS__) 348 #define kleave(FMT, ...) dbgprintk("<== %s()"FMT"", __func__, ##__VA_ARGS__) 349 #define kdebug(FMT, ...) dbgprintk(FMT, ##__VA_ARGS__) 350 351 #define kjournal(FMT, ...) no_printk(FMT, ##__VA_ARGS__) 352 353 #ifdef __KDEBUG 354 #define _enter(FMT, ...) kenter(FMT, ##__VA_ARGS__) 355 #define _leave(FMT, ...) kleave(FMT, ##__VA_ARGS__) 356 #define _debug(FMT, ...) kdebug(FMT, ##__VA_ARGS__) 357 358 #elif defined(CONFIG_FSCACHE_DEBUG) 359 #define _enter(FMT, ...) \ 360 do { \ 361 if (__do_kdebug(ENTER)) \ 362 kenter(FMT, ##__VA_ARGS__); \ 363 } while (0) 364 365 #define _leave(FMT, ...) \ 366 do { \ 367 if (__do_kdebug(LEAVE)) \ 368 kleave(FMT, ##__VA_ARGS__); \ 369 } while (0) 370 371 #define _debug(FMT, ...) \ 372 do { \ 373 if (__do_kdebug(DEBUG)) \ 374 kdebug(FMT, ##__VA_ARGS__); \ 375 } while (0) 376 377 #else 378 #define _enter(FMT, ...) no_printk("==> %s("FMT")", __func__, ##__VA_ARGS__) 379 #define _leave(FMT, ...) no_printk("<== %s()"FMT"", __func__, ##__VA_ARGS__) 380 #define _debug(FMT, ...) no_printk(FMT, ##__VA_ARGS__) 381 #endif 382 383 /* 384 * determine whether a particular optional debugging point should be logged 385 * - we need to go through three steps to persuade cpp to correctly join the 386 * shorthand in FSCACHE_DEBUG_LEVEL with its prefix 387 */ 388 #define ____do_kdebug(LEVEL, POINT) \ 389 unlikely((fscache_debug & \ 390 (FSCACHE_POINT_##POINT << (FSCACHE_DEBUG_ ## LEVEL * 3)))) 391 #define ___do_kdebug(LEVEL, POINT) \ 392 ____do_kdebug(LEVEL, POINT) 393 #define __do_kdebug(POINT) \ 394 ___do_kdebug(FSCACHE_DEBUG_LEVEL, POINT) 395 396 #define FSCACHE_DEBUG_CACHE 0 397 #define FSCACHE_DEBUG_COOKIE 1 398 #define FSCACHE_DEBUG_PAGE 2 399 #define FSCACHE_DEBUG_OPERATION 3 400 401 #define FSCACHE_POINT_ENTER 1 402 #define FSCACHE_POINT_LEAVE 2 403 #define FSCACHE_POINT_DEBUG 4 404 405 #ifndef FSCACHE_DEBUG_LEVEL 406 #define FSCACHE_DEBUG_LEVEL CACHE 407 #endif 408 409 /* 410 * assertions 411 */ 412 #if 1 /* defined(__KDEBUGALL) */ 413 414 #define ASSERT(X) \ 415 do { \ 416 if (unlikely(!(X))) { \ 417 pr_err("\n"); \ 418 pr_err("Assertion failed\n"); \ 419 BUG(); \ 420 } \ 421 } while (0) 422 423 #define ASSERTCMP(X, OP, Y) \ 424 do { \ 425 if (unlikely(!((X) OP (Y)))) { \ 426 pr_err("\n"); \ 427 pr_err("Assertion failed\n"); \ 428 pr_err("%lx " #OP " %lx is false\n", \ 429 (unsigned long)(X), (unsigned long)(Y)); \ 430 BUG(); \ 431 } \ 432 } while (0) 433 434 #define ASSERTIF(C, X) \ 435 do { \ 436 if (unlikely((C) && !(X))) { \ 437 pr_err("\n"); \ 438 pr_err("Assertion failed\n"); \ 439 BUG(); \ 440 } \ 441 } while (0) 442 443 #define ASSERTIFCMP(C, X, OP, Y) \ 444 do { \ 445 if (unlikely((C) && !((X) OP (Y)))) { \ 446 pr_err("\n"); \ 447 pr_err("Assertion failed\n"); \ 448 pr_err("%lx " #OP " %lx is false\n", \ 449 (unsigned long)(X), (unsigned long)(Y)); \ 450 BUG(); \ 451 } \ 452 } while (0) 453 454 #else 455 456 #define ASSERT(X) do {} while (0) 457 #define ASSERTCMP(X, OP, Y) do {} while (0) 458 #define ASSERTIF(C, X) do {} while (0) 459 #define ASSERTIFCMP(C, X, OP, Y) do {} while (0) 460 461 #endif /* assert or not */ 462