frontswap.c (8056B)
1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * Frontswap frontend 4 * 5 * This code provides the generic "frontend" layer to call a matching 6 * "backend" driver implementation of frontswap. See 7 * Documentation/vm/frontswap.rst for more information. 8 * 9 * Copyright (C) 2009-2012 Oracle Corp. All rights reserved. 10 * Author: Dan Magenheimer 11 */ 12 13#include <linux/mman.h> 14#include <linux/swap.h> 15#include <linux/swapops.h> 16#include <linux/security.h> 17#include <linux/module.h> 18#include <linux/debugfs.h> 19#include <linux/frontswap.h> 20#include <linux/swapfile.h> 21 22DEFINE_STATIC_KEY_FALSE(frontswap_enabled_key); 23 24/* 25 * frontswap_ops are added by frontswap_register_ops, and provide the 26 * frontswap "backend" implementation functions. Multiple implementations 27 * may be registered, but implementations can never deregister. This 28 * is a simple singly-linked list of all registered implementations. 29 */ 30static const struct frontswap_ops *frontswap_ops __read_mostly; 31 32#ifdef CONFIG_DEBUG_FS 33/* 34 * Counters available via /sys/kernel/debug/frontswap (if debugfs is 35 * properly configured). These are for information only so are not protected 36 * against increment races. 37 */ 38static u64 frontswap_loads; 39static u64 frontswap_succ_stores; 40static u64 frontswap_failed_stores; 41static u64 frontswap_invalidates; 42 43static inline void inc_frontswap_loads(void) 44{ 45 data_race(frontswap_loads++); 46} 47static inline void inc_frontswap_succ_stores(void) 48{ 49 data_race(frontswap_succ_stores++); 50} 51static inline void inc_frontswap_failed_stores(void) 52{ 53 data_race(frontswap_failed_stores++); 54} 55static inline void inc_frontswap_invalidates(void) 56{ 57 data_race(frontswap_invalidates++); 58} 59#else 60static inline void inc_frontswap_loads(void) { } 61static inline void inc_frontswap_succ_stores(void) { } 62static inline void inc_frontswap_failed_stores(void) { } 63static inline void inc_frontswap_invalidates(void) { } 64#endif 65 66/* 67 * Due to the asynchronous nature of the backends loading potentially 68 * _after_ the swap system has been activated, we have chokepoints 69 * on all frontswap functions to not call the backend until the backend 70 * has registered. 71 * 72 * This would not guards us against the user deciding to call swapoff right as 73 * we are calling the backend to initialize (so swapon is in action). 74 * Fortunately for us, the swapon_mutex has been taken by the callee so we are 75 * OK. The other scenario where calls to frontswap_store (called via 76 * swap_writepage) is racing with frontswap_invalidate_area (called via 77 * swapoff) is again guarded by the swap subsystem. 78 * 79 * While no backend is registered all calls to frontswap_[store|load| 80 * invalidate_area|invalidate_page] are ignored or fail. 81 * 82 * The time between the backend being registered and the swap file system 83 * calling the backend (via the frontswap_* functions) is indeterminate as 84 * frontswap_ops is not atomic_t (or a value guarded by a spinlock). 85 * That is OK as we are comfortable missing some of these calls to the newly 86 * registered backend. 87 * 88 * Obviously the opposite (unloading the backend) must be done after all 89 * the frontswap_[store|load|invalidate_area|invalidate_page] start 90 * ignoring or failing the requests. However, there is currently no way 91 * to unload a backend once it is registered. 92 */ 93 94/* 95 * Register operations for frontswap 96 */ 97int frontswap_register_ops(const struct frontswap_ops *ops) 98{ 99 if (frontswap_ops) 100 return -EINVAL; 101 102 frontswap_ops = ops; 103 static_branch_inc(&frontswap_enabled_key); 104 return 0; 105} 106 107/* 108 * Called when a swap device is swapon'd. 109 */ 110void frontswap_init(unsigned type, unsigned long *map) 111{ 112 struct swap_info_struct *sis = swap_info[type]; 113 114 VM_BUG_ON(sis == NULL); 115 116 /* 117 * p->frontswap is a bitmap that we MUST have to figure out which page 118 * has gone in frontswap. Without it there is no point of continuing. 119 */ 120 if (WARN_ON(!map)) 121 return; 122 /* 123 * Irregardless of whether the frontswap backend has been loaded 124 * before this function or it will be later, we _MUST_ have the 125 * p->frontswap set to something valid to work properly. 126 */ 127 frontswap_map_set(sis, map); 128 frontswap_ops->init(type); 129} 130 131static bool __frontswap_test(struct swap_info_struct *sis, 132 pgoff_t offset) 133{ 134 if (sis->frontswap_map) 135 return test_bit(offset, sis->frontswap_map); 136 return false; 137} 138 139static inline void __frontswap_set(struct swap_info_struct *sis, 140 pgoff_t offset) 141{ 142 set_bit(offset, sis->frontswap_map); 143 atomic_inc(&sis->frontswap_pages); 144} 145 146static inline void __frontswap_clear(struct swap_info_struct *sis, 147 pgoff_t offset) 148{ 149 clear_bit(offset, sis->frontswap_map); 150 atomic_dec(&sis->frontswap_pages); 151} 152 153/* 154 * "Store" data from a page to frontswap and associate it with the page's 155 * swaptype and offset. Page must be locked and in the swap cache. 156 * If frontswap already contains a page with matching swaptype and 157 * offset, the frontswap implementation may either overwrite the data and 158 * return success or invalidate the page from frontswap and return failure. 159 */ 160int __frontswap_store(struct page *page) 161{ 162 int ret = -1; 163 swp_entry_t entry = { .val = page_private(page), }; 164 int type = swp_type(entry); 165 struct swap_info_struct *sis = swap_info[type]; 166 pgoff_t offset = swp_offset(entry); 167 168 VM_BUG_ON(!frontswap_ops); 169 VM_BUG_ON(!PageLocked(page)); 170 VM_BUG_ON(sis == NULL); 171 172 /* 173 * If a dup, we must remove the old page first; we can't leave the 174 * old page no matter if the store of the new page succeeds or fails, 175 * and we can't rely on the new page replacing the old page as we may 176 * not store to the same implementation that contains the old page. 177 */ 178 if (__frontswap_test(sis, offset)) { 179 __frontswap_clear(sis, offset); 180 frontswap_ops->invalidate_page(type, offset); 181 } 182 183 ret = frontswap_ops->store(type, offset, page); 184 if (ret == 0) { 185 __frontswap_set(sis, offset); 186 inc_frontswap_succ_stores(); 187 } else { 188 inc_frontswap_failed_stores(); 189 } 190 191 return ret; 192} 193 194/* 195 * "Get" data from frontswap associated with swaptype and offset that were 196 * specified when the data was put to frontswap and use it to fill the 197 * specified page with data. Page must be locked and in the swap cache. 198 */ 199int __frontswap_load(struct page *page) 200{ 201 int ret = -1; 202 swp_entry_t entry = { .val = page_private(page), }; 203 int type = swp_type(entry); 204 struct swap_info_struct *sis = swap_info[type]; 205 pgoff_t offset = swp_offset(entry); 206 207 VM_BUG_ON(!frontswap_ops); 208 VM_BUG_ON(!PageLocked(page)); 209 VM_BUG_ON(sis == NULL); 210 211 if (!__frontswap_test(sis, offset)) 212 return -1; 213 214 /* Try loading from each implementation, until one succeeds. */ 215 ret = frontswap_ops->load(type, offset, page); 216 if (ret == 0) 217 inc_frontswap_loads(); 218 return ret; 219} 220 221/* 222 * Invalidate any data from frontswap associated with the specified swaptype 223 * and offset so that a subsequent "get" will fail. 224 */ 225void __frontswap_invalidate_page(unsigned type, pgoff_t offset) 226{ 227 struct swap_info_struct *sis = swap_info[type]; 228 229 VM_BUG_ON(!frontswap_ops); 230 VM_BUG_ON(sis == NULL); 231 232 if (!__frontswap_test(sis, offset)) 233 return; 234 235 frontswap_ops->invalidate_page(type, offset); 236 __frontswap_clear(sis, offset); 237 inc_frontswap_invalidates(); 238} 239 240/* 241 * Invalidate all data from frontswap associated with all offsets for the 242 * specified swaptype. 243 */ 244void __frontswap_invalidate_area(unsigned type) 245{ 246 struct swap_info_struct *sis = swap_info[type]; 247 248 VM_BUG_ON(!frontswap_ops); 249 VM_BUG_ON(sis == NULL); 250 251 if (sis->frontswap_map == NULL) 252 return; 253 254 frontswap_ops->invalidate_area(type); 255 atomic_set(&sis->frontswap_pages, 0); 256 bitmap_zero(sis->frontswap_map, sis->max); 257} 258 259static int __init init_frontswap(void) 260{ 261#ifdef CONFIG_DEBUG_FS 262 struct dentry *root = debugfs_create_dir("frontswap", NULL); 263 if (root == NULL) 264 return -ENXIO; 265 debugfs_create_u64("loads", 0444, root, &frontswap_loads); 266 debugfs_create_u64("succ_stores", 0444, root, &frontswap_succ_stores); 267 debugfs_create_u64("failed_stores", 0444, root, 268 &frontswap_failed_stores); 269 debugfs_create_u64("invalidates", 0444, root, &frontswap_invalidates); 270#endif 271 return 0; 272} 273 274module_init(init_frontswap);