userfaultfd_k.h (8646B)
1/* SPDX-License-Identifier: GPL-2.0 */ 2/* 3 * include/linux/userfaultfd_k.h 4 * 5 * Copyright (C) 2015 Red Hat, Inc. 6 * 7 */ 8 9#ifndef _LINUX_USERFAULTFD_K_H 10#define _LINUX_USERFAULTFD_K_H 11 12#ifdef CONFIG_USERFAULTFD 13 14#include <linux/userfaultfd.h> /* linux/include/uapi/linux/userfaultfd.h */ 15 16#include <linux/fcntl.h> 17#include <linux/mm.h> 18#include <linux/swap.h> 19#include <linux/swapops.h> 20#include <asm-generic/pgtable_uffd.h> 21#include <linux/hugetlb_inline.h> 22 23/* The set of all possible UFFD-related VM flags. */ 24#define __VM_UFFD_FLAGS (VM_UFFD_MISSING | VM_UFFD_WP | VM_UFFD_MINOR) 25 26/* 27 * CAREFUL: Check include/uapi/asm-generic/fcntl.h when defining 28 * new flags, since they might collide with O_* ones. We want 29 * to re-use O_* flags that couldn't possibly have a meaning 30 * from userfaultfd, in order to leave a free define-space for 31 * shared O_* flags. 32 */ 33#define UFFD_CLOEXEC O_CLOEXEC 34#define UFFD_NONBLOCK O_NONBLOCK 35 36#define UFFD_SHARED_FCNTL_FLAGS (O_CLOEXEC | O_NONBLOCK) 37#define UFFD_FLAGS_SET (EFD_SHARED_FCNTL_FLAGS) 38 39extern int sysctl_unprivileged_userfaultfd; 40 41extern vm_fault_t handle_userfault(struct vm_fault *vmf, unsigned long reason); 42 43/* 44 * The mode of operation for __mcopy_atomic and its helpers. 45 * 46 * This is almost an implementation detail (mcopy_atomic below doesn't take this 47 * as a parameter), but it's exposed here because memory-kind-specific 48 * implementations (e.g. hugetlbfs) need to know the mode of operation. 49 */ 50enum mcopy_atomic_mode { 51 /* A normal copy_from_user into the destination range. */ 52 MCOPY_ATOMIC_NORMAL, 53 /* Don't copy; map the destination range to the zero page. */ 54 MCOPY_ATOMIC_ZEROPAGE, 55 /* Just install pte(s) with the existing page(s) in the page cache. */ 56 MCOPY_ATOMIC_CONTINUE, 57}; 58 59extern int mfill_atomic_install_pte(struct mm_struct *dst_mm, pmd_t *dst_pmd, 60 struct vm_area_struct *dst_vma, 61 unsigned long dst_addr, struct page *page, 62 bool newly_allocated, bool wp_copy); 63 64extern ssize_t mcopy_atomic(struct mm_struct *dst_mm, unsigned long dst_start, 65 unsigned long src_start, unsigned long len, 66 atomic_t *mmap_changing, __u64 mode); 67extern ssize_t mfill_zeropage(struct mm_struct *dst_mm, 68 unsigned long dst_start, 69 unsigned long len, 70 atomic_t *mmap_changing); 71extern ssize_t mcopy_continue(struct mm_struct *dst_mm, unsigned long dst_start, 72 unsigned long len, atomic_t *mmap_changing); 73extern int mwriteprotect_range(struct mm_struct *dst_mm, 74 unsigned long start, unsigned long len, 75 bool enable_wp, atomic_t *mmap_changing); 76 77/* mm helpers */ 78static inline bool is_mergeable_vm_userfaultfd_ctx(struct vm_area_struct *vma, 79 struct vm_userfaultfd_ctx vm_ctx) 80{ 81 return vma->vm_userfaultfd_ctx.ctx == vm_ctx.ctx; 82} 83 84/* 85 * Never enable huge pmd sharing on some uffd registered vmas: 86 * 87 * - VM_UFFD_WP VMAs, because write protect information is per pgtable entry. 88 * 89 * - VM_UFFD_MINOR VMAs, because otherwise we would never get minor faults for 90 * VMAs which share huge pmds. (If you have two mappings to the same 91 * underlying pages, and fault in the non-UFFD-registered one with a write, 92 * with huge pmd sharing this would *also* setup the second UFFD-registered 93 * mapping, and we'd not get minor faults.) 94 */ 95static inline bool uffd_disable_huge_pmd_share(struct vm_area_struct *vma) 96{ 97 return vma->vm_flags & (VM_UFFD_WP | VM_UFFD_MINOR); 98} 99 100/* 101 * Don't do fault around for either WP or MINOR registered uffd range. For 102 * MINOR registered range, fault around will be a total disaster and ptes can 103 * be installed without notifications; for WP it should mostly be fine as long 104 * as the fault around checks for pte_none() before the installation, however 105 * to be super safe we just forbid it. 106 */ 107static inline bool uffd_disable_fault_around(struct vm_area_struct *vma) 108{ 109 return vma->vm_flags & (VM_UFFD_WP | VM_UFFD_MINOR); 110} 111 112static inline bool userfaultfd_missing(struct vm_area_struct *vma) 113{ 114 return vma->vm_flags & VM_UFFD_MISSING; 115} 116 117static inline bool userfaultfd_wp(struct vm_area_struct *vma) 118{ 119 return vma->vm_flags & VM_UFFD_WP; 120} 121 122static inline bool userfaultfd_minor(struct vm_area_struct *vma) 123{ 124 return vma->vm_flags & VM_UFFD_MINOR; 125} 126 127static inline bool userfaultfd_pte_wp(struct vm_area_struct *vma, 128 pte_t pte) 129{ 130 return userfaultfd_wp(vma) && pte_uffd_wp(pte); 131} 132 133static inline bool userfaultfd_huge_pmd_wp(struct vm_area_struct *vma, 134 pmd_t pmd) 135{ 136 return userfaultfd_wp(vma) && pmd_uffd_wp(pmd); 137} 138 139static inline bool userfaultfd_armed(struct vm_area_struct *vma) 140{ 141 return vma->vm_flags & __VM_UFFD_FLAGS; 142} 143 144static inline bool vma_can_userfault(struct vm_area_struct *vma, 145 unsigned long vm_flags) 146{ 147 if (vm_flags & VM_UFFD_MINOR) 148 return is_vm_hugetlb_page(vma) || vma_is_shmem(vma); 149 150#ifndef CONFIG_PTE_MARKER_UFFD_WP 151 /* 152 * If user requested uffd-wp but not enabled pte markers for 153 * uffd-wp, then shmem & hugetlbfs are not supported but only 154 * anonymous. 155 */ 156 if ((vm_flags & VM_UFFD_WP) && !vma_is_anonymous(vma)) 157 return false; 158#endif 159 return vma_is_anonymous(vma) || is_vm_hugetlb_page(vma) || 160 vma_is_shmem(vma); 161} 162 163extern int dup_userfaultfd(struct vm_area_struct *, struct list_head *); 164extern void dup_userfaultfd_complete(struct list_head *); 165 166extern void mremap_userfaultfd_prep(struct vm_area_struct *, 167 struct vm_userfaultfd_ctx *); 168extern void mremap_userfaultfd_complete(struct vm_userfaultfd_ctx *, 169 unsigned long from, unsigned long to, 170 unsigned long len); 171 172extern bool userfaultfd_remove(struct vm_area_struct *vma, 173 unsigned long start, 174 unsigned long end); 175 176extern int userfaultfd_unmap_prep(struct vm_area_struct *vma, 177 unsigned long start, unsigned long end, 178 struct list_head *uf); 179extern void userfaultfd_unmap_complete(struct mm_struct *mm, 180 struct list_head *uf); 181 182#else /* CONFIG_USERFAULTFD */ 183 184/* mm helpers */ 185static inline vm_fault_t handle_userfault(struct vm_fault *vmf, 186 unsigned long reason) 187{ 188 return VM_FAULT_SIGBUS; 189} 190 191static inline bool is_mergeable_vm_userfaultfd_ctx(struct vm_area_struct *vma, 192 struct vm_userfaultfd_ctx vm_ctx) 193{ 194 return true; 195} 196 197static inline bool userfaultfd_missing(struct vm_area_struct *vma) 198{ 199 return false; 200} 201 202static inline bool userfaultfd_wp(struct vm_area_struct *vma) 203{ 204 return false; 205} 206 207static inline bool userfaultfd_minor(struct vm_area_struct *vma) 208{ 209 return false; 210} 211 212static inline bool userfaultfd_pte_wp(struct vm_area_struct *vma, 213 pte_t pte) 214{ 215 return false; 216} 217 218static inline bool userfaultfd_huge_pmd_wp(struct vm_area_struct *vma, 219 pmd_t pmd) 220{ 221 return false; 222} 223 224 225static inline bool userfaultfd_armed(struct vm_area_struct *vma) 226{ 227 return false; 228} 229 230static inline int dup_userfaultfd(struct vm_area_struct *vma, 231 struct list_head *l) 232{ 233 return 0; 234} 235 236static inline void dup_userfaultfd_complete(struct list_head *l) 237{ 238} 239 240static inline void mremap_userfaultfd_prep(struct vm_area_struct *vma, 241 struct vm_userfaultfd_ctx *ctx) 242{ 243} 244 245static inline void mremap_userfaultfd_complete(struct vm_userfaultfd_ctx *ctx, 246 unsigned long from, 247 unsigned long to, 248 unsigned long len) 249{ 250} 251 252static inline bool userfaultfd_remove(struct vm_area_struct *vma, 253 unsigned long start, 254 unsigned long end) 255{ 256 return true; 257} 258 259static inline int userfaultfd_unmap_prep(struct vm_area_struct *vma, 260 unsigned long start, unsigned long end, 261 struct list_head *uf) 262{ 263 return 0; 264} 265 266static inline void userfaultfd_unmap_complete(struct mm_struct *mm, 267 struct list_head *uf) 268{ 269} 270 271static inline bool uffd_disable_fault_around(struct vm_area_struct *vma) 272{ 273 return false; 274} 275 276#endif /* CONFIG_USERFAULTFD */ 277 278static inline bool pte_marker_entry_uffd_wp(swp_entry_t entry) 279{ 280#ifdef CONFIG_PTE_MARKER_UFFD_WP 281 return is_pte_marker_entry(entry) && 282 (pte_marker_get(entry) & PTE_MARKER_UFFD_WP); 283#else 284 return false; 285#endif 286} 287 288static inline bool pte_marker_uffd_wp(pte_t pte) 289{ 290#ifdef CONFIG_PTE_MARKER_UFFD_WP 291 swp_entry_t entry; 292 293 if (!is_swap_pte(pte)) 294 return false; 295 296 entry = pte_to_swp_entry(pte); 297 298 return pte_marker_entry_uffd_wp(entry); 299#else 300 return false; 301#endif 302} 303 304/* 305 * Returns true if this is a swap pte and was uffd-wp wr-protected in either 306 * forms (pte marker or a normal swap pte), false otherwise. 307 */ 308static inline bool pte_swp_uffd_wp_any(pte_t pte) 309{ 310#ifdef CONFIG_PTE_MARKER_UFFD_WP 311 if (!is_swap_pte(pte)) 312 return false; 313 314 if (pte_swp_uffd_wp(pte)) 315 return true; 316 317 if (pte_marker_uffd_wp(pte)) 318 return true; 319#endif 320 return false; 321} 322 323#endif /* _LINUX_USERFAULTFD_K_H */