cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

memory.c (161944B)


      1// SPDX-License-Identifier: GPL-2.0-only
      2/*
      3 *  linux/mm/memory.c
      4 *
      5 *  Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
      6 */
      7
      8/*
      9 * demand-loading started 01.12.91 - seems it is high on the list of
     10 * things wanted, and it should be easy to implement. - Linus
     11 */
     12
     13/*
     14 * Ok, demand-loading was easy, shared pages a little bit tricker. Shared
     15 * pages started 02.12.91, seems to work. - Linus.
     16 *
     17 * Tested sharing by executing about 30 /bin/sh: under the old kernel it
     18 * would have taken more than the 6M I have free, but it worked well as
     19 * far as I could see.
     20 *
     21 * Also corrected some "invalidate()"s - I wasn't doing enough of them.
     22 */
     23
     24/*
     25 * Real VM (paging to/from disk) started 18.12.91. Much more work and
     26 * thought has to go into this. Oh, well..
     27 * 19.12.91  -  works, somewhat. Sometimes I get faults, don't know why.
     28 *		Found it. Everything seems to work now.
     29 * 20.12.91  -  Ok, making the swap-device changeable like the root.
     30 */
     31
     32/*
     33 * 05.04.94  -  Multi-page memory management added for v1.1.
     34 *              Idea by Alex Bligh (alex@cconcepts.co.uk)
     35 *
     36 * 16.07.99  -  Support of BIGMEM added by Gerhard Wichert, Siemens AG
     37 *		(Gerhard.Wichert@pdb.siemens.de)
     38 *
     39 * Aug/Sep 2004 Changed to four level page tables (Andi Kleen)
     40 */
     41
     42#include <linux/kernel_stat.h>
     43#include <linux/mm.h>
     44#include <linux/mm_inline.h>
     45#include <linux/sched/mm.h>
     46#include <linux/sched/coredump.h>
     47#include <linux/sched/numa_balancing.h>
     48#include <linux/sched/task.h>
     49#include <linux/hugetlb.h>
     50#include <linux/mman.h>
     51#include <linux/swap.h>
     52#include <linux/highmem.h>
     53#include <linux/pagemap.h>
     54#include <linux/memremap.h>
     55#include <linux/ksm.h>
     56#include <linux/rmap.h>
     57#include <linux/export.h>
     58#include <linux/delayacct.h>
     59#include <linux/init.h>
     60#include <linux/pfn_t.h>
     61#include <linux/writeback.h>
     62#include <linux/memcontrol.h>
     63#include <linux/mmu_notifier.h>
     64#include <linux/swapops.h>
     65#include <linux/elf.h>
     66#include <linux/gfp.h>
     67#include <linux/migrate.h>
     68#include <linux/string.h>
     69#include <linux/debugfs.h>
     70#include <linux/userfaultfd_k.h>
     71#include <linux/dax.h>
     72#include <linux/oom.h>
     73#include <linux/numa.h>
     74#include <linux/perf_event.h>
     75#include <linux/ptrace.h>
     76#include <linux/vmalloc.h>
     77
     78#include <trace/events/kmem.h>
     79
     80#include <asm/io.h>
     81#include <asm/mmu_context.h>
     82#include <asm/pgalloc.h>
     83#include <linux/uaccess.h>
     84#include <asm/tlb.h>
     85#include <asm/tlbflush.h>
     86
     87#include "pgalloc-track.h"
     88#include "internal.h"
     89#include "swap.h"
     90
     91#if defined(LAST_CPUPID_NOT_IN_PAGE_FLAGS) && !defined(CONFIG_COMPILE_TEST)
     92#warning Unfortunate NUMA and NUMA Balancing config, growing page-frame for last_cpupid.
     93#endif
     94
     95#ifndef CONFIG_NUMA
     96unsigned long max_mapnr;
     97EXPORT_SYMBOL(max_mapnr);
     98
     99struct page *mem_map;
    100EXPORT_SYMBOL(mem_map);
    101#endif
    102
    103static vm_fault_t do_fault(struct vm_fault *vmf);
    104
    105/*
    106 * A number of key systems in x86 including ioremap() rely on the assumption
    107 * that high_memory defines the upper bound on direct map memory, then end
    108 * of ZONE_NORMAL.  Under CONFIG_DISCONTIG this means that max_low_pfn and
    109 * highstart_pfn must be the same; there must be no gap between ZONE_NORMAL
    110 * and ZONE_HIGHMEM.
    111 */
    112void *high_memory;
    113EXPORT_SYMBOL(high_memory);
    114
    115/*
    116 * Randomize the address space (stacks, mmaps, brk, etc.).
    117 *
    118 * ( When CONFIG_COMPAT_BRK=y we exclude brk from randomization,
    119 *   as ancient (libc5 based) binaries can segfault. )
    120 */
    121int randomize_va_space __read_mostly =
    122#ifdef CONFIG_COMPAT_BRK
    123					1;
    124#else
    125					2;
    126#endif
    127
    128#ifndef arch_faults_on_old_pte
    129static inline bool arch_faults_on_old_pte(void)
    130{
    131	/*
    132	 * Those arches which don't have hw access flag feature need to
    133	 * implement their own helper. By default, "true" means pagefault
    134	 * will be hit on old pte.
    135	 */
    136	return true;
    137}
    138#endif
    139
    140#ifndef arch_wants_old_prefaulted_pte
    141static inline bool arch_wants_old_prefaulted_pte(void)
    142{
    143	/*
    144	 * Transitioning a PTE from 'old' to 'young' can be expensive on
    145	 * some architectures, even if it's performed in hardware. By
    146	 * default, "false" means prefaulted entries will be 'young'.
    147	 */
    148	return false;
    149}
    150#endif
    151
    152static int __init disable_randmaps(char *s)
    153{
    154	randomize_va_space = 0;
    155	return 1;
    156}
    157__setup("norandmaps", disable_randmaps);
    158
    159unsigned long zero_pfn __read_mostly;
    160EXPORT_SYMBOL(zero_pfn);
    161
    162unsigned long highest_memmap_pfn __read_mostly;
    163
    164/*
    165 * CONFIG_MMU architectures set up ZERO_PAGE in their paging_init()
    166 */
    167static int __init init_zero_pfn(void)
    168{
    169	zero_pfn = page_to_pfn(ZERO_PAGE(0));
    170	return 0;
    171}
    172early_initcall(init_zero_pfn);
    173
    174void mm_trace_rss_stat(struct mm_struct *mm, int member, long count)
    175{
    176	trace_rss_stat(mm, member, count);
    177}
    178
    179#if defined(SPLIT_RSS_COUNTING)
    180
    181void sync_mm_rss(struct mm_struct *mm)
    182{
    183	int i;
    184
    185	for (i = 0; i < NR_MM_COUNTERS; i++) {
    186		if (current->rss_stat.count[i]) {
    187			add_mm_counter(mm, i, current->rss_stat.count[i]);
    188			current->rss_stat.count[i] = 0;
    189		}
    190	}
    191	current->rss_stat.events = 0;
    192}
    193
    194static void add_mm_counter_fast(struct mm_struct *mm, int member, int val)
    195{
    196	struct task_struct *task = current;
    197
    198	if (likely(task->mm == mm))
    199		task->rss_stat.count[member] += val;
    200	else
    201		add_mm_counter(mm, member, val);
    202}
    203#define inc_mm_counter_fast(mm, member) add_mm_counter_fast(mm, member, 1)
    204#define dec_mm_counter_fast(mm, member) add_mm_counter_fast(mm, member, -1)
    205
    206/* sync counter once per 64 page faults */
    207#define TASK_RSS_EVENTS_THRESH	(64)
    208static void check_sync_rss_stat(struct task_struct *task)
    209{
    210	if (unlikely(task != current))
    211		return;
    212	if (unlikely(task->rss_stat.events++ > TASK_RSS_EVENTS_THRESH))
    213		sync_mm_rss(task->mm);
    214}
    215#else /* SPLIT_RSS_COUNTING */
    216
    217#define inc_mm_counter_fast(mm, member) inc_mm_counter(mm, member)
    218#define dec_mm_counter_fast(mm, member) dec_mm_counter(mm, member)
    219
    220static void check_sync_rss_stat(struct task_struct *task)
    221{
    222}
    223
    224#endif /* SPLIT_RSS_COUNTING */
    225
    226/*
    227 * Note: this doesn't free the actual pages themselves. That
    228 * has been handled earlier when unmapping all the memory regions.
    229 */
    230static void free_pte_range(struct mmu_gather *tlb, pmd_t *pmd,
    231			   unsigned long addr)
    232{
    233	pgtable_t token = pmd_pgtable(*pmd);
    234	pmd_clear(pmd);
    235	pte_free_tlb(tlb, token, addr);
    236	mm_dec_nr_ptes(tlb->mm);
    237}
    238
    239static inline void free_pmd_range(struct mmu_gather *tlb, pud_t *pud,
    240				unsigned long addr, unsigned long end,
    241				unsigned long floor, unsigned long ceiling)
    242{
    243	pmd_t *pmd;
    244	unsigned long next;
    245	unsigned long start;
    246
    247	start = addr;
    248	pmd = pmd_offset(pud, addr);
    249	do {
    250		next = pmd_addr_end(addr, end);
    251		if (pmd_none_or_clear_bad(pmd))
    252			continue;
    253		free_pte_range(tlb, pmd, addr);
    254	} while (pmd++, addr = next, addr != end);
    255
    256	start &= PUD_MASK;
    257	if (start < floor)
    258		return;
    259	if (ceiling) {
    260		ceiling &= PUD_MASK;
    261		if (!ceiling)
    262			return;
    263	}
    264	if (end - 1 > ceiling - 1)
    265		return;
    266
    267	pmd = pmd_offset(pud, start);
    268	pud_clear(pud);
    269	pmd_free_tlb(tlb, pmd, start);
    270	mm_dec_nr_pmds(tlb->mm);
    271}
    272
    273static inline void free_pud_range(struct mmu_gather *tlb, p4d_t *p4d,
    274				unsigned long addr, unsigned long end,
    275				unsigned long floor, unsigned long ceiling)
    276{
    277	pud_t *pud;
    278	unsigned long next;
    279	unsigned long start;
    280
    281	start = addr;
    282	pud = pud_offset(p4d, addr);
    283	do {
    284		next = pud_addr_end(addr, end);
    285		if (pud_none_or_clear_bad(pud))
    286			continue;
    287		free_pmd_range(tlb, pud, addr, next, floor, ceiling);
    288	} while (pud++, addr = next, addr != end);
    289
    290	start &= P4D_MASK;
    291	if (start < floor)
    292		return;
    293	if (ceiling) {
    294		ceiling &= P4D_MASK;
    295		if (!ceiling)
    296			return;
    297	}
    298	if (end - 1 > ceiling - 1)
    299		return;
    300
    301	pud = pud_offset(p4d, start);
    302	p4d_clear(p4d);
    303	pud_free_tlb(tlb, pud, start);
    304	mm_dec_nr_puds(tlb->mm);
    305}
    306
    307static inline void free_p4d_range(struct mmu_gather *tlb, pgd_t *pgd,
    308				unsigned long addr, unsigned long end,
    309				unsigned long floor, unsigned long ceiling)
    310{
    311	p4d_t *p4d;
    312	unsigned long next;
    313	unsigned long start;
    314
    315	start = addr;
    316	p4d = p4d_offset(pgd, addr);
    317	do {
    318		next = p4d_addr_end(addr, end);
    319		if (p4d_none_or_clear_bad(p4d))
    320			continue;
    321		free_pud_range(tlb, p4d, addr, next, floor, ceiling);
    322	} while (p4d++, addr = next, addr != end);
    323
    324	start &= PGDIR_MASK;
    325	if (start < floor)
    326		return;
    327	if (ceiling) {
    328		ceiling &= PGDIR_MASK;
    329		if (!ceiling)
    330			return;
    331	}
    332	if (end - 1 > ceiling - 1)
    333		return;
    334
    335	p4d = p4d_offset(pgd, start);
    336	pgd_clear(pgd);
    337	p4d_free_tlb(tlb, p4d, start);
    338}
    339
    340/*
    341 * This function frees user-level page tables of a process.
    342 */
    343void free_pgd_range(struct mmu_gather *tlb,
    344			unsigned long addr, unsigned long end,
    345			unsigned long floor, unsigned long ceiling)
    346{
    347	pgd_t *pgd;
    348	unsigned long next;
    349
    350	/*
    351	 * The next few lines have given us lots of grief...
    352	 *
    353	 * Why are we testing PMD* at this top level?  Because often
    354	 * there will be no work to do at all, and we'd prefer not to
    355	 * go all the way down to the bottom just to discover that.
    356	 *
    357	 * Why all these "- 1"s?  Because 0 represents both the bottom
    358	 * of the address space and the top of it (using -1 for the
    359	 * top wouldn't help much: the masks would do the wrong thing).
    360	 * The rule is that addr 0 and floor 0 refer to the bottom of
    361	 * the address space, but end 0 and ceiling 0 refer to the top
    362	 * Comparisons need to use "end - 1" and "ceiling - 1" (though
    363	 * that end 0 case should be mythical).
    364	 *
    365	 * Wherever addr is brought up or ceiling brought down, we must
    366	 * be careful to reject "the opposite 0" before it confuses the
    367	 * subsequent tests.  But what about where end is brought down
    368	 * by PMD_SIZE below? no, end can't go down to 0 there.
    369	 *
    370	 * Whereas we round start (addr) and ceiling down, by different
    371	 * masks at different levels, in order to test whether a table
    372	 * now has no other vmas using it, so can be freed, we don't
    373	 * bother to round floor or end up - the tests don't need that.
    374	 */
    375
    376	addr &= PMD_MASK;
    377	if (addr < floor) {
    378		addr += PMD_SIZE;
    379		if (!addr)
    380			return;
    381	}
    382	if (ceiling) {
    383		ceiling &= PMD_MASK;
    384		if (!ceiling)
    385			return;
    386	}
    387	if (end - 1 > ceiling - 1)
    388		end -= PMD_SIZE;
    389	if (addr > end - 1)
    390		return;
    391	/*
    392	 * We add page table cache pages with PAGE_SIZE,
    393	 * (see pte_free_tlb()), flush the tlb if we need
    394	 */
    395	tlb_change_page_size(tlb, PAGE_SIZE);
    396	pgd = pgd_offset(tlb->mm, addr);
    397	do {
    398		next = pgd_addr_end(addr, end);
    399		if (pgd_none_or_clear_bad(pgd))
    400			continue;
    401		free_p4d_range(tlb, pgd, addr, next, floor, ceiling);
    402	} while (pgd++, addr = next, addr != end);
    403}
    404
    405void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *vma,
    406		unsigned long floor, unsigned long ceiling)
    407{
    408	while (vma) {
    409		struct vm_area_struct *next = vma->vm_next;
    410		unsigned long addr = vma->vm_start;
    411
    412		/*
    413		 * Hide vma from rmap and truncate_pagecache before freeing
    414		 * pgtables
    415		 */
    416		unlink_anon_vmas(vma);
    417		unlink_file_vma(vma);
    418
    419		if (is_vm_hugetlb_page(vma)) {
    420			hugetlb_free_pgd_range(tlb, addr, vma->vm_end,
    421				floor, next ? next->vm_start : ceiling);
    422		} else {
    423			/*
    424			 * Optimization: gather nearby vmas into one call down
    425			 */
    426			while (next && next->vm_start <= vma->vm_end + PMD_SIZE
    427			       && !is_vm_hugetlb_page(next)) {
    428				vma = next;
    429				next = vma->vm_next;
    430				unlink_anon_vmas(vma);
    431				unlink_file_vma(vma);
    432			}
    433			free_pgd_range(tlb, addr, vma->vm_end,
    434				floor, next ? next->vm_start : ceiling);
    435		}
    436		vma = next;
    437	}
    438}
    439
    440void pmd_install(struct mm_struct *mm, pmd_t *pmd, pgtable_t *pte)
    441{
    442	spinlock_t *ptl = pmd_lock(mm, pmd);
    443
    444	if (likely(pmd_none(*pmd))) {	/* Has another populated it ? */
    445		mm_inc_nr_ptes(mm);
    446		/*
    447		 * Ensure all pte setup (eg. pte page lock and page clearing) are
    448		 * visible before the pte is made visible to other CPUs by being
    449		 * put into page tables.
    450		 *
    451		 * The other side of the story is the pointer chasing in the page
    452		 * table walking code (when walking the page table without locking;
    453		 * ie. most of the time). Fortunately, these data accesses consist
    454		 * of a chain of data-dependent loads, meaning most CPUs (alpha
    455		 * being the notable exception) will already guarantee loads are
    456		 * seen in-order. See the alpha page table accessors for the
    457		 * smp_rmb() barriers in page table walking code.
    458		 */
    459		smp_wmb(); /* Could be smp_wmb__xxx(before|after)_spin_lock */
    460		pmd_populate(mm, pmd, *pte);
    461		*pte = NULL;
    462	}
    463	spin_unlock(ptl);
    464}
    465
    466int __pte_alloc(struct mm_struct *mm, pmd_t *pmd)
    467{
    468	pgtable_t new = pte_alloc_one(mm);
    469	if (!new)
    470		return -ENOMEM;
    471
    472	pmd_install(mm, pmd, &new);
    473	if (new)
    474		pte_free(mm, new);
    475	return 0;
    476}
    477
    478int __pte_alloc_kernel(pmd_t *pmd)
    479{
    480	pte_t *new = pte_alloc_one_kernel(&init_mm);
    481	if (!new)
    482		return -ENOMEM;
    483
    484	spin_lock(&init_mm.page_table_lock);
    485	if (likely(pmd_none(*pmd))) {	/* Has another populated it ? */
    486		smp_wmb(); /* See comment in pmd_install() */
    487		pmd_populate_kernel(&init_mm, pmd, new);
    488		new = NULL;
    489	}
    490	spin_unlock(&init_mm.page_table_lock);
    491	if (new)
    492		pte_free_kernel(&init_mm, new);
    493	return 0;
    494}
    495
    496static inline void init_rss_vec(int *rss)
    497{
    498	memset(rss, 0, sizeof(int) * NR_MM_COUNTERS);
    499}
    500
    501static inline void add_mm_rss_vec(struct mm_struct *mm, int *rss)
    502{
    503	int i;
    504
    505	if (current->mm == mm)
    506		sync_mm_rss(mm);
    507	for (i = 0; i < NR_MM_COUNTERS; i++)
    508		if (rss[i])
    509			add_mm_counter(mm, i, rss[i]);
    510}
    511
    512/*
    513 * This function is called to print an error when a bad pte
    514 * is found. For example, we might have a PFN-mapped pte in
    515 * a region that doesn't allow it.
    516 *
    517 * The calling function must still handle the error.
    518 */
    519static void print_bad_pte(struct vm_area_struct *vma, unsigned long addr,
    520			  pte_t pte, struct page *page)
    521{
    522	pgd_t *pgd = pgd_offset(vma->vm_mm, addr);
    523	p4d_t *p4d = p4d_offset(pgd, addr);
    524	pud_t *pud = pud_offset(p4d, addr);
    525	pmd_t *pmd = pmd_offset(pud, addr);
    526	struct address_space *mapping;
    527	pgoff_t index;
    528	static unsigned long resume;
    529	static unsigned long nr_shown;
    530	static unsigned long nr_unshown;
    531
    532	/*
    533	 * Allow a burst of 60 reports, then keep quiet for that minute;
    534	 * or allow a steady drip of one report per second.
    535	 */
    536	if (nr_shown == 60) {
    537		if (time_before(jiffies, resume)) {
    538			nr_unshown++;
    539			return;
    540		}
    541		if (nr_unshown) {
    542			pr_alert("BUG: Bad page map: %lu messages suppressed\n",
    543				 nr_unshown);
    544			nr_unshown = 0;
    545		}
    546		nr_shown = 0;
    547	}
    548	if (nr_shown++ == 0)
    549		resume = jiffies + 60 * HZ;
    550
    551	mapping = vma->vm_file ? vma->vm_file->f_mapping : NULL;
    552	index = linear_page_index(vma, addr);
    553
    554	pr_alert("BUG: Bad page map in process %s  pte:%08llx pmd:%08llx\n",
    555		 current->comm,
    556		 (long long)pte_val(pte), (long long)pmd_val(*pmd));
    557	if (page)
    558		dump_page(page, "bad pte");
    559	pr_alert("addr:%px vm_flags:%08lx anon_vma:%px mapping:%px index:%lx\n",
    560		 (void *)addr, vma->vm_flags, vma->anon_vma, mapping, index);
    561	pr_alert("file:%pD fault:%ps mmap:%ps read_folio:%ps\n",
    562		 vma->vm_file,
    563		 vma->vm_ops ? vma->vm_ops->fault : NULL,
    564		 vma->vm_file ? vma->vm_file->f_op->mmap : NULL,
    565		 mapping ? mapping->a_ops->read_folio : NULL);
    566	dump_stack();
    567	add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
    568}
    569
    570/*
    571 * vm_normal_page -- This function gets the "struct page" associated with a pte.
    572 *
    573 * "Special" mappings do not wish to be associated with a "struct page" (either
    574 * it doesn't exist, or it exists but they don't want to touch it). In this
    575 * case, NULL is returned here. "Normal" mappings do have a struct page.
    576 *
    577 * There are 2 broad cases. Firstly, an architecture may define a pte_special()
    578 * pte bit, in which case this function is trivial. Secondly, an architecture
    579 * may not have a spare pte bit, which requires a more complicated scheme,
    580 * described below.
    581 *
    582 * A raw VM_PFNMAP mapping (ie. one that is not COWed) is always considered a
    583 * special mapping (even if there are underlying and valid "struct pages").
    584 * COWed pages of a VM_PFNMAP are always normal.
    585 *
    586 * The way we recognize COWed pages within VM_PFNMAP mappings is through the
    587 * rules set up by "remap_pfn_range()": the vma will have the VM_PFNMAP bit
    588 * set, and the vm_pgoff will point to the first PFN mapped: thus every special
    589 * mapping will always honor the rule
    590 *
    591 *	pfn_of_page == vma->vm_pgoff + ((addr - vma->vm_start) >> PAGE_SHIFT)
    592 *
    593 * And for normal mappings this is false.
    594 *
    595 * This restricts such mappings to be a linear translation from virtual address
    596 * to pfn. To get around this restriction, we allow arbitrary mappings so long
    597 * as the vma is not a COW mapping; in that case, we know that all ptes are
    598 * special (because none can have been COWed).
    599 *
    600 *
    601 * In order to support COW of arbitrary special mappings, we have VM_MIXEDMAP.
    602 *
    603 * VM_MIXEDMAP mappings can likewise contain memory with or without "struct
    604 * page" backing, however the difference is that _all_ pages with a struct
    605 * page (that is, those where pfn_valid is true) are refcounted and considered
    606 * normal pages by the VM. The disadvantage is that pages are refcounted
    607 * (which can be slower and simply not an option for some PFNMAP users). The
    608 * advantage is that we don't have to follow the strict linearity rule of
    609 * PFNMAP mappings in order to support COWable mappings.
    610 *
    611 */
    612struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr,
    613			    pte_t pte)
    614{
    615	unsigned long pfn = pte_pfn(pte);
    616
    617	if (IS_ENABLED(CONFIG_ARCH_HAS_PTE_SPECIAL)) {
    618		if (likely(!pte_special(pte)))
    619			goto check_pfn;
    620		if (vma->vm_ops && vma->vm_ops->find_special_page)
    621			return vma->vm_ops->find_special_page(vma, addr);
    622		if (vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP))
    623			return NULL;
    624		if (is_zero_pfn(pfn))
    625			return NULL;
    626		if (pte_devmap(pte))
    627			return NULL;
    628
    629		print_bad_pte(vma, addr, pte, NULL);
    630		return NULL;
    631	}
    632
    633	/* !CONFIG_ARCH_HAS_PTE_SPECIAL case follows: */
    634
    635	if (unlikely(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP))) {
    636		if (vma->vm_flags & VM_MIXEDMAP) {
    637			if (!pfn_valid(pfn))
    638				return NULL;
    639			goto out;
    640		} else {
    641			unsigned long off;
    642			off = (addr - vma->vm_start) >> PAGE_SHIFT;
    643			if (pfn == vma->vm_pgoff + off)
    644				return NULL;
    645			if (!is_cow_mapping(vma->vm_flags))
    646				return NULL;
    647		}
    648	}
    649
    650	if (is_zero_pfn(pfn))
    651		return NULL;
    652
    653check_pfn:
    654	if (unlikely(pfn > highest_memmap_pfn)) {
    655		print_bad_pte(vma, addr, pte, NULL);
    656		return NULL;
    657	}
    658
    659	/*
    660	 * NOTE! We still have PageReserved() pages in the page tables.
    661	 * eg. VDSO mappings can cause them to exist.
    662	 */
    663out:
    664	return pfn_to_page(pfn);
    665}
    666
    667#ifdef CONFIG_TRANSPARENT_HUGEPAGE
    668struct page *vm_normal_page_pmd(struct vm_area_struct *vma, unsigned long addr,
    669				pmd_t pmd)
    670{
    671	unsigned long pfn = pmd_pfn(pmd);
    672
    673	/*
    674	 * There is no pmd_special() but there may be special pmds, e.g.
    675	 * in a direct-access (dax) mapping, so let's just replicate the
    676	 * !CONFIG_ARCH_HAS_PTE_SPECIAL case from vm_normal_page() here.
    677	 */
    678	if (unlikely(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP))) {
    679		if (vma->vm_flags & VM_MIXEDMAP) {
    680			if (!pfn_valid(pfn))
    681				return NULL;
    682			goto out;
    683		} else {
    684			unsigned long off;
    685			off = (addr - vma->vm_start) >> PAGE_SHIFT;
    686			if (pfn == vma->vm_pgoff + off)
    687				return NULL;
    688			if (!is_cow_mapping(vma->vm_flags))
    689				return NULL;
    690		}
    691	}
    692
    693	if (pmd_devmap(pmd))
    694		return NULL;
    695	if (is_huge_zero_pmd(pmd))
    696		return NULL;
    697	if (unlikely(pfn > highest_memmap_pfn))
    698		return NULL;
    699
    700	/*
    701	 * NOTE! We still have PageReserved() pages in the page tables.
    702	 * eg. VDSO mappings can cause them to exist.
    703	 */
    704out:
    705	return pfn_to_page(pfn);
    706}
    707#endif
    708
    709static void restore_exclusive_pte(struct vm_area_struct *vma,
    710				  struct page *page, unsigned long address,
    711				  pte_t *ptep)
    712{
    713	pte_t pte;
    714	swp_entry_t entry;
    715
    716	pte = pte_mkold(mk_pte(page, READ_ONCE(vma->vm_page_prot)));
    717	if (pte_swp_soft_dirty(*ptep))
    718		pte = pte_mksoft_dirty(pte);
    719
    720	entry = pte_to_swp_entry(*ptep);
    721	if (pte_swp_uffd_wp(*ptep))
    722		pte = pte_mkuffd_wp(pte);
    723	else if (is_writable_device_exclusive_entry(entry))
    724		pte = maybe_mkwrite(pte_mkdirty(pte), vma);
    725
    726	VM_BUG_ON(pte_write(pte) && !(PageAnon(page) && PageAnonExclusive(page)));
    727
    728	/*
    729	 * No need to take a page reference as one was already
    730	 * created when the swap entry was made.
    731	 */
    732	if (PageAnon(page))
    733		page_add_anon_rmap(page, vma, address, RMAP_NONE);
    734	else
    735		/*
    736		 * Currently device exclusive access only supports anonymous
    737		 * memory so the entry shouldn't point to a filebacked page.
    738		 */
    739		WARN_ON_ONCE(!PageAnon(page));
    740
    741	set_pte_at(vma->vm_mm, address, ptep, pte);
    742
    743	/*
    744	 * No need to invalidate - it was non-present before. However
    745	 * secondary CPUs may have mappings that need invalidating.
    746	 */
    747	update_mmu_cache(vma, address, ptep);
    748}
    749
    750/*
    751 * Tries to restore an exclusive pte if the page lock can be acquired without
    752 * sleeping.
    753 */
    754static int
    755try_restore_exclusive_pte(pte_t *src_pte, struct vm_area_struct *vma,
    756			unsigned long addr)
    757{
    758	swp_entry_t entry = pte_to_swp_entry(*src_pte);
    759	struct page *page = pfn_swap_entry_to_page(entry);
    760
    761	if (trylock_page(page)) {
    762		restore_exclusive_pte(vma, page, addr, src_pte);
    763		unlock_page(page);
    764		return 0;
    765	}
    766
    767	return -EBUSY;
    768}
    769
    770/*
    771 * copy one vm_area from one task to the other. Assumes the page tables
    772 * already present in the new task to be cleared in the whole range
    773 * covered by this vma.
    774 */
    775
    776static unsigned long
    777copy_nonpresent_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm,
    778		pte_t *dst_pte, pte_t *src_pte, struct vm_area_struct *dst_vma,
    779		struct vm_area_struct *src_vma, unsigned long addr, int *rss)
    780{
    781	unsigned long vm_flags = dst_vma->vm_flags;
    782	pte_t pte = *src_pte;
    783	struct page *page;
    784	swp_entry_t entry = pte_to_swp_entry(pte);
    785
    786	if (likely(!non_swap_entry(entry))) {
    787		if (swap_duplicate(entry) < 0)
    788			return -EIO;
    789
    790		/* make sure dst_mm is on swapoff's mmlist. */
    791		if (unlikely(list_empty(&dst_mm->mmlist))) {
    792			spin_lock(&mmlist_lock);
    793			if (list_empty(&dst_mm->mmlist))
    794				list_add(&dst_mm->mmlist,
    795						&src_mm->mmlist);
    796			spin_unlock(&mmlist_lock);
    797		}
    798		/* Mark the swap entry as shared. */
    799		if (pte_swp_exclusive(*src_pte)) {
    800			pte = pte_swp_clear_exclusive(*src_pte);
    801			set_pte_at(src_mm, addr, src_pte, pte);
    802		}
    803		rss[MM_SWAPENTS]++;
    804	} else if (is_migration_entry(entry)) {
    805		page = pfn_swap_entry_to_page(entry);
    806
    807		rss[mm_counter(page)]++;
    808
    809		if (!is_readable_migration_entry(entry) &&
    810				is_cow_mapping(vm_flags)) {
    811			/*
    812			 * COW mappings require pages in both parent and child
    813			 * to be set to read. A previously exclusive entry is
    814			 * now shared.
    815			 */
    816			entry = make_readable_migration_entry(
    817							swp_offset(entry));
    818			pte = swp_entry_to_pte(entry);
    819			if (pte_swp_soft_dirty(*src_pte))
    820				pte = pte_swp_mksoft_dirty(pte);
    821			if (pte_swp_uffd_wp(*src_pte))
    822				pte = pte_swp_mkuffd_wp(pte);
    823			set_pte_at(src_mm, addr, src_pte, pte);
    824		}
    825	} else if (is_device_private_entry(entry)) {
    826		page = pfn_swap_entry_to_page(entry);
    827
    828		/*
    829		 * Update rss count even for unaddressable pages, as
    830		 * they should treated just like normal pages in this
    831		 * respect.
    832		 *
    833		 * We will likely want to have some new rss counters
    834		 * for unaddressable pages, at some point. But for now
    835		 * keep things as they are.
    836		 */
    837		get_page(page);
    838		rss[mm_counter(page)]++;
    839		/* Cannot fail as these pages cannot get pinned. */
    840		BUG_ON(page_try_dup_anon_rmap(page, false, src_vma));
    841
    842		/*
    843		 * We do not preserve soft-dirty information, because so
    844		 * far, checkpoint/restore is the only feature that
    845		 * requires that. And checkpoint/restore does not work
    846		 * when a device driver is involved (you cannot easily
    847		 * save and restore device driver state).
    848		 */
    849		if (is_writable_device_private_entry(entry) &&
    850		    is_cow_mapping(vm_flags)) {
    851			entry = make_readable_device_private_entry(
    852							swp_offset(entry));
    853			pte = swp_entry_to_pte(entry);
    854			if (pte_swp_uffd_wp(*src_pte))
    855				pte = pte_swp_mkuffd_wp(pte);
    856			set_pte_at(src_mm, addr, src_pte, pte);
    857		}
    858	} else if (is_device_exclusive_entry(entry)) {
    859		/*
    860		 * Make device exclusive entries present by restoring the
    861		 * original entry then copying as for a present pte. Device
    862		 * exclusive entries currently only support private writable
    863		 * (ie. COW) mappings.
    864		 */
    865		VM_BUG_ON(!is_cow_mapping(src_vma->vm_flags));
    866		if (try_restore_exclusive_pte(src_pte, src_vma, addr))
    867			return -EBUSY;
    868		return -ENOENT;
    869	} else if (is_pte_marker_entry(entry)) {
    870		/*
    871		 * We're copying the pgtable should only because dst_vma has
    872		 * uffd-wp enabled, do sanity check.
    873		 */
    874		WARN_ON_ONCE(!userfaultfd_wp(dst_vma));
    875		set_pte_at(dst_mm, addr, dst_pte, pte);
    876		return 0;
    877	}
    878	if (!userfaultfd_wp(dst_vma))
    879		pte = pte_swp_clear_uffd_wp(pte);
    880	set_pte_at(dst_mm, addr, dst_pte, pte);
    881	return 0;
    882}
    883
    884/*
    885 * Copy a present and normal page.
    886 *
    887 * NOTE! The usual case is that this isn't required;
    888 * instead, the caller can just increase the page refcount
    889 * and re-use the pte the traditional way.
    890 *
    891 * And if we need a pre-allocated page but don't yet have
    892 * one, return a negative error to let the preallocation
    893 * code know so that it can do so outside the page table
    894 * lock.
    895 */
    896static inline int
    897copy_present_page(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
    898		  pte_t *dst_pte, pte_t *src_pte, unsigned long addr, int *rss,
    899		  struct page **prealloc, struct page *page)
    900{
    901	struct page *new_page;
    902	pte_t pte;
    903
    904	new_page = *prealloc;
    905	if (!new_page)
    906		return -EAGAIN;
    907
    908	/*
    909	 * We have a prealloc page, all good!  Take it
    910	 * over and copy the page & arm it.
    911	 */
    912	*prealloc = NULL;
    913	copy_user_highpage(new_page, page, addr, src_vma);
    914	__SetPageUptodate(new_page);
    915	page_add_new_anon_rmap(new_page, dst_vma, addr);
    916	lru_cache_add_inactive_or_unevictable(new_page, dst_vma);
    917	rss[mm_counter(new_page)]++;
    918
    919	/* All done, just insert the new page copy in the child */
    920	pte = mk_pte(new_page, dst_vma->vm_page_prot);
    921	pte = maybe_mkwrite(pte_mkdirty(pte), dst_vma);
    922	if (userfaultfd_pte_wp(dst_vma, *src_pte))
    923		/* Uffd-wp needs to be delivered to dest pte as well */
    924		pte = pte_wrprotect(pte_mkuffd_wp(pte));
    925	set_pte_at(dst_vma->vm_mm, addr, dst_pte, pte);
    926	return 0;
    927}
    928
    929/*
    930 * Copy one pte.  Returns 0 if succeeded, or -EAGAIN if one preallocated page
    931 * is required to copy this pte.
    932 */
    933static inline int
    934copy_present_pte(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
    935		 pte_t *dst_pte, pte_t *src_pte, unsigned long addr, int *rss,
    936		 struct page **prealloc)
    937{
    938	struct mm_struct *src_mm = src_vma->vm_mm;
    939	unsigned long vm_flags = src_vma->vm_flags;
    940	pte_t pte = *src_pte;
    941	struct page *page;
    942
    943	page = vm_normal_page(src_vma, addr, pte);
    944	if (page && PageAnon(page)) {
    945		/*
    946		 * If this page may have been pinned by the parent process,
    947		 * copy the page immediately for the child so that we'll always
    948		 * guarantee the pinned page won't be randomly replaced in the
    949		 * future.
    950		 */
    951		get_page(page);
    952		if (unlikely(page_try_dup_anon_rmap(page, false, src_vma))) {
    953			/* Page maybe pinned, we have to copy. */
    954			put_page(page);
    955			return copy_present_page(dst_vma, src_vma, dst_pte, src_pte,
    956						 addr, rss, prealloc, page);
    957		}
    958		rss[mm_counter(page)]++;
    959	} else if (page) {
    960		get_page(page);
    961		page_dup_file_rmap(page, false);
    962		rss[mm_counter(page)]++;
    963	}
    964
    965	/*
    966	 * If it's a COW mapping, write protect it both
    967	 * in the parent and the child
    968	 */
    969	if (is_cow_mapping(vm_flags) && pte_write(pte)) {
    970		ptep_set_wrprotect(src_mm, addr, src_pte);
    971		pte = pte_wrprotect(pte);
    972	}
    973	VM_BUG_ON(page && PageAnon(page) && PageAnonExclusive(page));
    974
    975	/*
    976	 * If it's a shared mapping, mark it clean in
    977	 * the child
    978	 */
    979	if (vm_flags & VM_SHARED)
    980		pte = pte_mkclean(pte);
    981	pte = pte_mkold(pte);
    982
    983	if (!userfaultfd_wp(dst_vma))
    984		pte = pte_clear_uffd_wp(pte);
    985
    986	set_pte_at(dst_vma->vm_mm, addr, dst_pte, pte);
    987	return 0;
    988}
    989
    990static inline struct page *
    991page_copy_prealloc(struct mm_struct *src_mm, struct vm_area_struct *vma,
    992		   unsigned long addr)
    993{
    994	struct page *new_page;
    995
    996	new_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, addr);
    997	if (!new_page)
    998		return NULL;
    999
   1000	if (mem_cgroup_charge(page_folio(new_page), src_mm, GFP_KERNEL)) {
   1001		put_page(new_page);
   1002		return NULL;
   1003	}
   1004	cgroup_throttle_swaprate(new_page, GFP_KERNEL);
   1005
   1006	return new_page;
   1007}
   1008
   1009static int
   1010copy_pte_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
   1011	       pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long addr,
   1012	       unsigned long end)
   1013{
   1014	struct mm_struct *dst_mm = dst_vma->vm_mm;
   1015	struct mm_struct *src_mm = src_vma->vm_mm;
   1016	pte_t *orig_src_pte, *orig_dst_pte;
   1017	pte_t *src_pte, *dst_pte;
   1018	spinlock_t *src_ptl, *dst_ptl;
   1019	int progress, ret = 0;
   1020	int rss[NR_MM_COUNTERS];
   1021	swp_entry_t entry = (swp_entry_t){0};
   1022	struct page *prealloc = NULL;
   1023
   1024again:
   1025	progress = 0;
   1026	init_rss_vec(rss);
   1027
   1028	dst_pte = pte_alloc_map_lock(dst_mm, dst_pmd, addr, &dst_ptl);
   1029	if (!dst_pte) {
   1030		ret = -ENOMEM;
   1031		goto out;
   1032	}
   1033	src_pte = pte_offset_map(src_pmd, addr);
   1034	src_ptl = pte_lockptr(src_mm, src_pmd);
   1035	spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
   1036	orig_src_pte = src_pte;
   1037	orig_dst_pte = dst_pte;
   1038	arch_enter_lazy_mmu_mode();
   1039
   1040	do {
   1041		/*
   1042		 * We are holding two locks at this point - either of them
   1043		 * could generate latencies in another task on another CPU.
   1044		 */
   1045		if (progress >= 32) {
   1046			progress = 0;
   1047			if (need_resched() ||
   1048			    spin_needbreak(src_ptl) || spin_needbreak(dst_ptl))
   1049				break;
   1050		}
   1051		if (pte_none(*src_pte)) {
   1052			progress++;
   1053			continue;
   1054		}
   1055		if (unlikely(!pte_present(*src_pte))) {
   1056			ret = copy_nonpresent_pte(dst_mm, src_mm,
   1057						  dst_pte, src_pte,
   1058						  dst_vma, src_vma,
   1059						  addr, rss);
   1060			if (ret == -EIO) {
   1061				entry = pte_to_swp_entry(*src_pte);
   1062				break;
   1063			} else if (ret == -EBUSY) {
   1064				break;
   1065			} else if (!ret) {
   1066				progress += 8;
   1067				continue;
   1068			}
   1069
   1070			/*
   1071			 * Device exclusive entry restored, continue by copying
   1072			 * the now present pte.
   1073			 */
   1074			WARN_ON_ONCE(ret != -ENOENT);
   1075		}
   1076		/* copy_present_pte() will clear `*prealloc' if consumed */
   1077		ret = copy_present_pte(dst_vma, src_vma, dst_pte, src_pte,
   1078				       addr, rss, &prealloc);
   1079		/*
   1080		 * If we need a pre-allocated page for this pte, drop the
   1081		 * locks, allocate, and try again.
   1082		 */
   1083		if (unlikely(ret == -EAGAIN))
   1084			break;
   1085		if (unlikely(prealloc)) {
   1086			/*
   1087			 * pre-alloc page cannot be reused by next time so as
   1088			 * to strictly follow mempolicy (e.g., alloc_page_vma()
   1089			 * will allocate page according to address).  This
   1090			 * could only happen if one pinned pte changed.
   1091			 */
   1092			put_page(prealloc);
   1093			prealloc = NULL;
   1094		}
   1095		progress += 8;
   1096	} while (dst_pte++, src_pte++, addr += PAGE_SIZE, addr != end);
   1097
   1098	arch_leave_lazy_mmu_mode();
   1099	spin_unlock(src_ptl);
   1100	pte_unmap(orig_src_pte);
   1101	add_mm_rss_vec(dst_mm, rss);
   1102	pte_unmap_unlock(orig_dst_pte, dst_ptl);
   1103	cond_resched();
   1104
   1105	if (ret == -EIO) {
   1106		VM_WARN_ON_ONCE(!entry.val);
   1107		if (add_swap_count_continuation(entry, GFP_KERNEL) < 0) {
   1108			ret = -ENOMEM;
   1109			goto out;
   1110		}
   1111		entry.val = 0;
   1112	} else if (ret == -EBUSY) {
   1113		goto out;
   1114	} else if (ret ==  -EAGAIN) {
   1115		prealloc = page_copy_prealloc(src_mm, src_vma, addr);
   1116		if (!prealloc)
   1117			return -ENOMEM;
   1118	} else if (ret) {
   1119		VM_WARN_ON_ONCE(1);
   1120	}
   1121
   1122	/* We've captured and resolved the error. Reset, try again. */
   1123	ret = 0;
   1124
   1125	if (addr != end)
   1126		goto again;
   1127out:
   1128	if (unlikely(prealloc))
   1129		put_page(prealloc);
   1130	return ret;
   1131}
   1132
   1133static inline int
   1134copy_pmd_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
   1135	       pud_t *dst_pud, pud_t *src_pud, unsigned long addr,
   1136	       unsigned long end)
   1137{
   1138	struct mm_struct *dst_mm = dst_vma->vm_mm;
   1139	struct mm_struct *src_mm = src_vma->vm_mm;
   1140	pmd_t *src_pmd, *dst_pmd;
   1141	unsigned long next;
   1142
   1143	dst_pmd = pmd_alloc(dst_mm, dst_pud, addr);
   1144	if (!dst_pmd)
   1145		return -ENOMEM;
   1146	src_pmd = pmd_offset(src_pud, addr);
   1147	do {
   1148		next = pmd_addr_end(addr, end);
   1149		if (is_swap_pmd(*src_pmd) || pmd_trans_huge(*src_pmd)
   1150			|| pmd_devmap(*src_pmd)) {
   1151			int err;
   1152			VM_BUG_ON_VMA(next-addr != HPAGE_PMD_SIZE, src_vma);
   1153			err = copy_huge_pmd(dst_mm, src_mm, dst_pmd, src_pmd,
   1154					    addr, dst_vma, src_vma);
   1155			if (err == -ENOMEM)
   1156				return -ENOMEM;
   1157			if (!err)
   1158				continue;
   1159			/* fall through */
   1160		}
   1161		if (pmd_none_or_clear_bad(src_pmd))
   1162			continue;
   1163		if (copy_pte_range(dst_vma, src_vma, dst_pmd, src_pmd,
   1164				   addr, next))
   1165			return -ENOMEM;
   1166	} while (dst_pmd++, src_pmd++, addr = next, addr != end);
   1167	return 0;
   1168}
   1169
   1170static inline int
   1171copy_pud_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
   1172	       p4d_t *dst_p4d, p4d_t *src_p4d, unsigned long addr,
   1173	       unsigned long end)
   1174{
   1175	struct mm_struct *dst_mm = dst_vma->vm_mm;
   1176	struct mm_struct *src_mm = src_vma->vm_mm;
   1177	pud_t *src_pud, *dst_pud;
   1178	unsigned long next;
   1179
   1180	dst_pud = pud_alloc(dst_mm, dst_p4d, addr);
   1181	if (!dst_pud)
   1182		return -ENOMEM;
   1183	src_pud = pud_offset(src_p4d, addr);
   1184	do {
   1185		next = pud_addr_end(addr, end);
   1186		if (pud_trans_huge(*src_pud) || pud_devmap(*src_pud)) {
   1187			int err;
   1188
   1189			VM_BUG_ON_VMA(next-addr != HPAGE_PUD_SIZE, src_vma);
   1190			err = copy_huge_pud(dst_mm, src_mm,
   1191					    dst_pud, src_pud, addr, src_vma);
   1192			if (err == -ENOMEM)
   1193				return -ENOMEM;
   1194			if (!err)
   1195				continue;
   1196			/* fall through */
   1197		}
   1198		if (pud_none_or_clear_bad(src_pud))
   1199			continue;
   1200		if (copy_pmd_range(dst_vma, src_vma, dst_pud, src_pud,
   1201				   addr, next))
   1202			return -ENOMEM;
   1203	} while (dst_pud++, src_pud++, addr = next, addr != end);
   1204	return 0;
   1205}
   1206
   1207static inline int
   1208copy_p4d_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
   1209	       pgd_t *dst_pgd, pgd_t *src_pgd, unsigned long addr,
   1210	       unsigned long end)
   1211{
   1212	struct mm_struct *dst_mm = dst_vma->vm_mm;
   1213	p4d_t *src_p4d, *dst_p4d;
   1214	unsigned long next;
   1215
   1216	dst_p4d = p4d_alloc(dst_mm, dst_pgd, addr);
   1217	if (!dst_p4d)
   1218		return -ENOMEM;
   1219	src_p4d = p4d_offset(src_pgd, addr);
   1220	do {
   1221		next = p4d_addr_end(addr, end);
   1222		if (p4d_none_or_clear_bad(src_p4d))
   1223			continue;
   1224		if (copy_pud_range(dst_vma, src_vma, dst_p4d, src_p4d,
   1225				   addr, next))
   1226			return -ENOMEM;
   1227	} while (dst_p4d++, src_p4d++, addr = next, addr != end);
   1228	return 0;
   1229}
   1230
   1231/*
   1232 * Return true if the vma needs to copy the pgtable during this fork().  Return
   1233 * false when we can speed up fork() by allowing lazy page faults later until
   1234 * when the child accesses the memory range.
   1235 */
   1236static bool
   1237vma_needs_copy(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma)
   1238{
   1239	/*
   1240	 * Always copy pgtables when dst_vma has uffd-wp enabled even if it's
   1241	 * file-backed (e.g. shmem). Because when uffd-wp is enabled, pgtable
   1242	 * contains uffd-wp protection information, that's something we can't
   1243	 * retrieve from page cache, and skip copying will lose those info.
   1244	 */
   1245	if (userfaultfd_wp(dst_vma))
   1246		return true;
   1247
   1248	if (src_vma->vm_flags & (VM_HUGETLB | VM_PFNMAP | VM_MIXEDMAP))
   1249		return true;
   1250
   1251	if (src_vma->anon_vma)
   1252		return true;
   1253
   1254	/*
   1255	 * Don't copy ptes where a page fault will fill them correctly.  Fork
   1256	 * becomes much lighter when there are big shared or private readonly
   1257	 * mappings. The tradeoff is that copy_page_range is more efficient
   1258	 * than faulting.
   1259	 */
   1260	return false;
   1261}
   1262
   1263int
   1264copy_page_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma)
   1265{
   1266	pgd_t *src_pgd, *dst_pgd;
   1267	unsigned long next;
   1268	unsigned long addr = src_vma->vm_start;
   1269	unsigned long end = src_vma->vm_end;
   1270	struct mm_struct *dst_mm = dst_vma->vm_mm;
   1271	struct mm_struct *src_mm = src_vma->vm_mm;
   1272	struct mmu_notifier_range range;
   1273	bool is_cow;
   1274	int ret;
   1275
   1276	if (!vma_needs_copy(dst_vma, src_vma))
   1277		return 0;
   1278
   1279	if (is_vm_hugetlb_page(src_vma))
   1280		return copy_hugetlb_page_range(dst_mm, src_mm, dst_vma, src_vma);
   1281
   1282	if (unlikely(src_vma->vm_flags & VM_PFNMAP)) {
   1283		/*
   1284		 * We do not free on error cases below as remove_vma
   1285		 * gets called on error from higher level routine
   1286		 */
   1287		ret = track_pfn_copy(src_vma);
   1288		if (ret)
   1289			return ret;
   1290	}
   1291
   1292	/*
   1293	 * We need to invalidate the secondary MMU mappings only when
   1294	 * there could be a permission downgrade on the ptes of the
   1295	 * parent mm. And a permission downgrade will only happen if
   1296	 * is_cow_mapping() returns true.
   1297	 */
   1298	is_cow = is_cow_mapping(src_vma->vm_flags);
   1299
   1300	if (is_cow) {
   1301		mmu_notifier_range_init(&range, MMU_NOTIFY_PROTECTION_PAGE,
   1302					0, src_vma, src_mm, addr, end);
   1303		mmu_notifier_invalidate_range_start(&range);
   1304		/*
   1305		 * Disabling preemption is not needed for the write side, as
   1306		 * the read side doesn't spin, but goes to the mmap_lock.
   1307		 *
   1308		 * Use the raw variant of the seqcount_t write API to avoid
   1309		 * lockdep complaining about preemptibility.
   1310		 */
   1311		mmap_assert_write_locked(src_mm);
   1312		raw_write_seqcount_begin(&src_mm->write_protect_seq);
   1313	}
   1314
   1315	ret = 0;
   1316	dst_pgd = pgd_offset(dst_mm, addr);
   1317	src_pgd = pgd_offset(src_mm, addr);
   1318	do {
   1319		next = pgd_addr_end(addr, end);
   1320		if (pgd_none_or_clear_bad(src_pgd))
   1321			continue;
   1322		if (unlikely(copy_p4d_range(dst_vma, src_vma, dst_pgd, src_pgd,
   1323					    addr, next))) {
   1324			ret = -ENOMEM;
   1325			break;
   1326		}
   1327	} while (dst_pgd++, src_pgd++, addr = next, addr != end);
   1328
   1329	if (is_cow) {
   1330		raw_write_seqcount_end(&src_mm->write_protect_seq);
   1331		mmu_notifier_invalidate_range_end(&range);
   1332	}
   1333	return ret;
   1334}
   1335
   1336/*
   1337 * Parameter block passed down to zap_pte_range in exceptional cases.
   1338 */
   1339struct zap_details {
   1340	struct folio *single_folio;	/* Locked folio to be unmapped */
   1341	bool even_cows;			/* Zap COWed private pages too? */
   1342	zap_flags_t zap_flags;		/* Extra flags for zapping */
   1343};
   1344
   1345/* Whether we should zap all COWed (private) pages too */
   1346static inline bool should_zap_cows(struct zap_details *details)
   1347{
   1348	/* By default, zap all pages */
   1349	if (!details)
   1350		return true;
   1351
   1352	/* Or, we zap COWed pages only if the caller wants to */
   1353	return details->even_cows;
   1354}
   1355
   1356/* Decides whether we should zap this page with the page pointer specified */
   1357static inline bool should_zap_page(struct zap_details *details, struct page *page)
   1358{
   1359	/* If we can make a decision without *page.. */
   1360	if (should_zap_cows(details))
   1361		return true;
   1362
   1363	/* E.g. the caller passes NULL for the case of a zero page */
   1364	if (!page)
   1365		return true;
   1366
   1367	/* Otherwise we should only zap non-anon pages */
   1368	return !PageAnon(page);
   1369}
   1370
   1371static inline bool zap_drop_file_uffd_wp(struct zap_details *details)
   1372{
   1373	if (!details)
   1374		return false;
   1375
   1376	return details->zap_flags & ZAP_FLAG_DROP_MARKER;
   1377}
   1378
   1379/*
   1380 * This function makes sure that we'll replace the none pte with an uffd-wp
   1381 * swap special pte marker when necessary. Must be with the pgtable lock held.
   1382 */
   1383static inline void
   1384zap_install_uffd_wp_if_needed(struct vm_area_struct *vma,
   1385			      unsigned long addr, pte_t *pte,
   1386			      struct zap_details *details, pte_t pteval)
   1387{
   1388	if (zap_drop_file_uffd_wp(details))
   1389		return;
   1390
   1391	pte_install_uffd_wp_if_needed(vma, addr, pte, pteval);
   1392}
   1393
   1394static unsigned long zap_pte_range(struct mmu_gather *tlb,
   1395				struct vm_area_struct *vma, pmd_t *pmd,
   1396				unsigned long addr, unsigned long end,
   1397				struct zap_details *details)
   1398{
   1399	struct mm_struct *mm = tlb->mm;
   1400	int force_flush = 0;
   1401	int rss[NR_MM_COUNTERS];
   1402	spinlock_t *ptl;
   1403	pte_t *start_pte;
   1404	pte_t *pte;
   1405	swp_entry_t entry;
   1406
   1407	tlb_change_page_size(tlb, PAGE_SIZE);
   1408again:
   1409	init_rss_vec(rss);
   1410	start_pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
   1411	pte = start_pte;
   1412	flush_tlb_batched_pending(mm);
   1413	arch_enter_lazy_mmu_mode();
   1414	do {
   1415		pte_t ptent = *pte;
   1416		struct page *page;
   1417
   1418		if (pte_none(ptent))
   1419			continue;
   1420
   1421		if (need_resched())
   1422			break;
   1423
   1424		if (pte_present(ptent)) {
   1425			page = vm_normal_page(vma, addr, ptent);
   1426			if (unlikely(!should_zap_page(details, page)))
   1427				continue;
   1428			ptent = ptep_get_and_clear_full(mm, addr, pte,
   1429							tlb->fullmm);
   1430			tlb_remove_tlb_entry(tlb, pte, addr);
   1431			zap_install_uffd_wp_if_needed(vma, addr, pte, details,
   1432						      ptent);
   1433			if (unlikely(!page))
   1434				continue;
   1435
   1436			if (!PageAnon(page)) {
   1437				if (pte_dirty(ptent)) {
   1438					force_flush = 1;
   1439					set_page_dirty(page);
   1440				}
   1441				if (pte_young(ptent) &&
   1442				    likely(!(vma->vm_flags & VM_SEQ_READ)))
   1443					mark_page_accessed(page);
   1444			}
   1445			rss[mm_counter(page)]--;
   1446			page_remove_rmap(page, vma, false);
   1447			if (unlikely(page_mapcount(page) < 0))
   1448				print_bad_pte(vma, addr, ptent, page);
   1449			if (unlikely(__tlb_remove_page(tlb, page))) {
   1450				force_flush = 1;
   1451				addr += PAGE_SIZE;
   1452				break;
   1453			}
   1454			continue;
   1455		}
   1456
   1457		entry = pte_to_swp_entry(ptent);
   1458		if (is_device_private_entry(entry) ||
   1459		    is_device_exclusive_entry(entry)) {
   1460			page = pfn_swap_entry_to_page(entry);
   1461			if (unlikely(!should_zap_page(details, page)))
   1462				continue;
   1463			/*
   1464			 * Both device private/exclusive mappings should only
   1465			 * work with anonymous page so far, so we don't need to
   1466			 * consider uffd-wp bit when zap. For more information,
   1467			 * see zap_install_uffd_wp_if_needed().
   1468			 */
   1469			WARN_ON_ONCE(!vma_is_anonymous(vma));
   1470			rss[mm_counter(page)]--;
   1471			if (is_device_private_entry(entry))
   1472				page_remove_rmap(page, vma, false);
   1473			put_page(page);
   1474		} else if (!non_swap_entry(entry)) {
   1475			/* Genuine swap entry, hence a private anon page */
   1476			if (!should_zap_cows(details))
   1477				continue;
   1478			rss[MM_SWAPENTS]--;
   1479			if (unlikely(!free_swap_and_cache(entry)))
   1480				print_bad_pte(vma, addr, ptent, NULL);
   1481		} else if (is_migration_entry(entry)) {
   1482			page = pfn_swap_entry_to_page(entry);
   1483			if (!should_zap_page(details, page))
   1484				continue;
   1485			rss[mm_counter(page)]--;
   1486		} else if (pte_marker_entry_uffd_wp(entry)) {
   1487			/* Only drop the uffd-wp marker if explicitly requested */
   1488			if (!zap_drop_file_uffd_wp(details))
   1489				continue;
   1490		} else if (is_hwpoison_entry(entry) ||
   1491			   is_swapin_error_entry(entry)) {
   1492			if (!should_zap_cows(details))
   1493				continue;
   1494		} else {
   1495			/* We should have covered all the swap entry types */
   1496			WARN_ON_ONCE(1);
   1497		}
   1498		pte_clear_not_present_full(mm, addr, pte, tlb->fullmm);
   1499		zap_install_uffd_wp_if_needed(vma, addr, pte, details, ptent);
   1500	} while (pte++, addr += PAGE_SIZE, addr != end);
   1501
   1502	add_mm_rss_vec(mm, rss);
   1503	arch_leave_lazy_mmu_mode();
   1504
   1505	/* Do the actual TLB flush before dropping ptl */
   1506	if (force_flush)
   1507		tlb_flush_mmu_tlbonly(tlb);
   1508	pte_unmap_unlock(start_pte, ptl);
   1509
   1510	/*
   1511	 * If we forced a TLB flush (either due to running out of
   1512	 * batch buffers or because we needed to flush dirty TLB
   1513	 * entries before releasing the ptl), free the batched
   1514	 * memory too. Restart if we didn't do everything.
   1515	 */
   1516	if (force_flush) {
   1517		force_flush = 0;
   1518		tlb_flush_mmu(tlb);
   1519	}
   1520
   1521	if (addr != end) {
   1522		cond_resched();
   1523		goto again;
   1524	}
   1525
   1526	return addr;
   1527}
   1528
   1529static inline unsigned long zap_pmd_range(struct mmu_gather *tlb,
   1530				struct vm_area_struct *vma, pud_t *pud,
   1531				unsigned long addr, unsigned long end,
   1532				struct zap_details *details)
   1533{
   1534	pmd_t *pmd;
   1535	unsigned long next;
   1536
   1537	pmd = pmd_offset(pud, addr);
   1538	do {
   1539		next = pmd_addr_end(addr, end);
   1540		if (is_swap_pmd(*pmd) || pmd_trans_huge(*pmd) || pmd_devmap(*pmd)) {
   1541			if (next - addr != HPAGE_PMD_SIZE)
   1542				__split_huge_pmd(vma, pmd, addr, false, NULL);
   1543			else if (zap_huge_pmd(tlb, vma, pmd, addr))
   1544				goto next;
   1545			/* fall through */
   1546		} else if (details && details->single_folio &&
   1547			   folio_test_pmd_mappable(details->single_folio) &&
   1548			   next - addr == HPAGE_PMD_SIZE && pmd_none(*pmd)) {
   1549			spinlock_t *ptl = pmd_lock(tlb->mm, pmd);
   1550			/*
   1551			 * Take and drop THP pmd lock so that we cannot return
   1552			 * prematurely, while zap_huge_pmd() has cleared *pmd,
   1553			 * but not yet decremented compound_mapcount().
   1554			 */
   1555			spin_unlock(ptl);
   1556		}
   1557
   1558		/*
   1559		 * Here there can be other concurrent MADV_DONTNEED or
   1560		 * trans huge page faults running, and if the pmd is
   1561		 * none or trans huge it can change under us. This is
   1562		 * because MADV_DONTNEED holds the mmap_lock in read
   1563		 * mode.
   1564		 */
   1565		if (pmd_none_or_trans_huge_or_clear_bad(pmd))
   1566			goto next;
   1567		next = zap_pte_range(tlb, vma, pmd, addr, next, details);
   1568next:
   1569		cond_resched();
   1570	} while (pmd++, addr = next, addr != end);
   1571
   1572	return addr;
   1573}
   1574
   1575static inline unsigned long zap_pud_range(struct mmu_gather *tlb,
   1576				struct vm_area_struct *vma, p4d_t *p4d,
   1577				unsigned long addr, unsigned long end,
   1578				struct zap_details *details)
   1579{
   1580	pud_t *pud;
   1581	unsigned long next;
   1582
   1583	pud = pud_offset(p4d, addr);
   1584	do {
   1585		next = pud_addr_end(addr, end);
   1586		if (pud_trans_huge(*pud) || pud_devmap(*pud)) {
   1587			if (next - addr != HPAGE_PUD_SIZE) {
   1588				mmap_assert_locked(tlb->mm);
   1589				split_huge_pud(vma, pud, addr);
   1590			} else if (zap_huge_pud(tlb, vma, pud, addr))
   1591				goto next;
   1592			/* fall through */
   1593		}
   1594		if (pud_none_or_clear_bad(pud))
   1595			continue;
   1596		next = zap_pmd_range(tlb, vma, pud, addr, next, details);
   1597next:
   1598		cond_resched();
   1599	} while (pud++, addr = next, addr != end);
   1600
   1601	return addr;
   1602}
   1603
   1604static inline unsigned long zap_p4d_range(struct mmu_gather *tlb,
   1605				struct vm_area_struct *vma, pgd_t *pgd,
   1606				unsigned long addr, unsigned long end,
   1607				struct zap_details *details)
   1608{
   1609	p4d_t *p4d;
   1610	unsigned long next;
   1611
   1612	p4d = p4d_offset(pgd, addr);
   1613	do {
   1614		next = p4d_addr_end(addr, end);
   1615		if (p4d_none_or_clear_bad(p4d))
   1616			continue;
   1617		next = zap_pud_range(tlb, vma, p4d, addr, next, details);
   1618	} while (p4d++, addr = next, addr != end);
   1619
   1620	return addr;
   1621}
   1622
   1623void unmap_page_range(struct mmu_gather *tlb,
   1624			     struct vm_area_struct *vma,
   1625			     unsigned long addr, unsigned long end,
   1626			     struct zap_details *details)
   1627{
   1628	pgd_t *pgd;
   1629	unsigned long next;
   1630
   1631	BUG_ON(addr >= end);
   1632	tlb_start_vma(tlb, vma);
   1633	pgd = pgd_offset(vma->vm_mm, addr);
   1634	do {
   1635		next = pgd_addr_end(addr, end);
   1636		if (pgd_none_or_clear_bad(pgd))
   1637			continue;
   1638		next = zap_p4d_range(tlb, vma, pgd, addr, next, details);
   1639	} while (pgd++, addr = next, addr != end);
   1640	tlb_end_vma(tlb, vma);
   1641}
   1642
   1643
   1644static void unmap_single_vma(struct mmu_gather *tlb,
   1645		struct vm_area_struct *vma, unsigned long start_addr,
   1646		unsigned long end_addr,
   1647		struct zap_details *details)
   1648{
   1649	unsigned long start = max(vma->vm_start, start_addr);
   1650	unsigned long end;
   1651
   1652	if (start >= vma->vm_end)
   1653		return;
   1654	end = min(vma->vm_end, end_addr);
   1655	if (end <= vma->vm_start)
   1656		return;
   1657
   1658	if (vma->vm_file)
   1659		uprobe_munmap(vma, start, end);
   1660
   1661	if (unlikely(vma->vm_flags & VM_PFNMAP))
   1662		untrack_pfn(vma, 0, 0);
   1663
   1664	if (start != end) {
   1665		if (unlikely(is_vm_hugetlb_page(vma))) {
   1666			/*
   1667			 * It is undesirable to test vma->vm_file as it
   1668			 * should be non-null for valid hugetlb area.
   1669			 * However, vm_file will be NULL in the error
   1670			 * cleanup path of mmap_region. When
   1671			 * hugetlbfs ->mmap method fails,
   1672			 * mmap_region() nullifies vma->vm_file
   1673			 * before calling this function to clean up.
   1674			 * Since no pte has actually been setup, it is
   1675			 * safe to do nothing in this case.
   1676			 */
   1677			if (vma->vm_file) {
   1678				zap_flags_t zap_flags = details ?
   1679				    details->zap_flags : 0;
   1680				i_mmap_lock_write(vma->vm_file->f_mapping);
   1681				__unmap_hugepage_range_final(tlb, vma, start, end,
   1682							     NULL, zap_flags);
   1683				i_mmap_unlock_write(vma->vm_file->f_mapping);
   1684			}
   1685		} else
   1686			unmap_page_range(tlb, vma, start, end, details);
   1687	}
   1688}
   1689
   1690/**
   1691 * unmap_vmas - unmap a range of memory covered by a list of vma's
   1692 * @tlb: address of the caller's struct mmu_gather
   1693 * @vma: the starting vma
   1694 * @start_addr: virtual address at which to start unmapping
   1695 * @end_addr: virtual address at which to end unmapping
   1696 *
   1697 * Unmap all pages in the vma list.
   1698 *
   1699 * Only addresses between `start' and `end' will be unmapped.
   1700 *
   1701 * The VMA list must be sorted in ascending virtual address order.
   1702 *
   1703 * unmap_vmas() assumes that the caller will flush the whole unmapped address
   1704 * range after unmap_vmas() returns.  So the only responsibility here is to
   1705 * ensure that any thus-far unmapped pages are flushed before unmap_vmas()
   1706 * drops the lock and schedules.
   1707 */
   1708void unmap_vmas(struct mmu_gather *tlb,
   1709		struct vm_area_struct *vma, unsigned long start_addr,
   1710		unsigned long end_addr)
   1711{
   1712	struct mmu_notifier_range range;
   1713	struct zap_details details = {
   1714		.zap_flags = ZAP_FLAG_DROP_MARKER,
   1715		/* Careful - we need to zap private pages too! */
   1716		.even_cows = true,
   1717	};
   1718
   1719	mmu_notifier_range_init(&range, MMU_NOTIFY_UNMAP, 0, vma, vma->vm_mm,
   1720				start_addr, end_addr);
   1721	mmu_notifier_invalidate_range_start(&range);
   1722	for ( ; vma && vma->vm_start < end_addr; vma = vma->vm_next)
   1723		unmap_single_vma(tlb, vma, start_addr, end_addr, &details);
   1724	mmu_notifier_invalidate_range_end(&range);
   1725}
   1726
   1727/**
   1728 * zap_page_range - remove user pages in a given range
   1729 * @vma: vm_area_struct holding the applicable pages
   1730 * @start: starting address of pages to zap
   1731 * @size: number of bytes to zap
   1732 *
   1733 * Caller must protect the VMA list
   1734 */
   1735void zap_page_range(struct vm_area_struct *vma, unsigned long start,
   1736		unsigned long size)
   1737{
   1738	struct mmu_notifier_range range;
   1739	struct mmu_gather tlb;
   1740
   1741	lru_add_drain();
   1742	mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, vma->vm_mm,
   1743				start, start + size);
   1744	tlb_gather_mmu(&tlb, vma->vm_mm);
   1745	update_hiwater_rss(vma->vm_mm);
   1746	mmu_notifier_invalidate_range_start(&range);
   1747	for ( ; vma && vma->vm_start < range.end; vma = vma->vm_next)
   1748		unmap_single_vma(&tlb, vma, start, range.end, NULL);
   1749	mmu_notifier_invalidate_range_end(&range);
   1750	tlb_finish_mmu(&tlb);
   1751}
   1752
   1753/**
   1754 * zap_page_range_single - remove user pages in a given range
   1755 * @vma: vm_area_struct holding the applicable pages
   1756 * @address: starting address of pages to zap
   1757 * @size: number of bytes to zap
   1758 * @details: details of shared cache invalidation
   1759 *
   1760 * The range must fit into one VMA.
   1761 */
   1762static void zap_page_range_single(struct vm_area_struct *vma, unsigned long address,
   1763		unsigned long size, struct zap_details *details)
   1764{
   1765	struct mmu_notifier_range range;
   1766	struct mmu_gather tlb;
   1767
   1768	lru_add_drain();
   1769	mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, vma->vm_mm,
   1770				address, address + size);
   1771	tlb_gather_mmu(&tlb, vma->vm_mm);
   1772	update_hiwater_rss(vma->vm_mm);
   1773	mmu_notifier_invalidate_range_start(&range);
   1774	unmap_single_vma(&tlb, vma, address, range.end, details);
   1775	mmu_notifier_invalidate_range_end(&range);
   1776	tlb_finish_mmu(&tlb);
   1777}
   1778
   1779/**
   1780 * zap_vma_ptes - remove ptes mapping the vma
   1781 * @vma: vm_area_struct holding ptes to be zapped
   1782 * @address: starting address of pages to zap
   1783 * @size: number of bytes to zap
   1784 *
   1785 * This function only unmaps ptes assigned to VM_PFNMAP vmas.
   1786 *
   1787 * The entire address range must be fully contained within the vma.
   1788 *
   1789 */
   1790void zap_vma_ptes(struct vm_area_struct *vma, unsigned long address,
   1791		unsigned long size)
   1792{
   1793	if (!range_in_vma(vma, address, address + size) ||
   1794	    		!(vma->vm_flags & VM_PFNMAP))
   1795		return;
   1796
   1797	zap_page_range_single(vma, address, size, NULL);
   1798}
   1799EXPORT_SYMBOL_GPL(zap_vma_ptes);
   1800
   1801static pmd_t *walk_to_pmd(struct mm_struct *mm, unsigned long addr)
   1802{
   1803	pgd_t *pgd;
   1804	p4d_t *p4d;
   1805	pud_t *pud;
   1806	pmd_t *pmd;
   1807
   1808	pgd = pgd_offset(mm, addr);
   1809	p4d = p4d_alloc(mm, pgd, addr);
   1810	if (!p4d)
   1811		return NULL;
   1812	pud = pud_alloc(mm, p4d, addr);
   1813	if (!pud)
   1814		return NULL;
   1815	pmd = pmd_alloc(mm, pud, addr);
   1816	if (!pmd)
   1817		return NULL;
   1818
   1819	VM_BUG_ON(pmd_trans_huge(*pmd));
   1820	return pmd;
   1821}
   1822
   1823pte_t *__get_locked_pte(struct mm_struct *mm, unsigned long addr,
   1824			spinlock_t **ptl)
   1825{
   1826	pmd_t *pmd = walk_to_pmd(mm, addr);
   1827
   1828	if (!pmd)
   1829		return NULL;
   1830	return pte_alloc_map_lock(mm, pmd, addr, ptl);
   1831}
   1832
   1833static int validate_page_before_insert(struct page *page)
   1834{
   1835	if (PageAnon(page) || PageSlab(page) || page_has_type(page))
   1836		return -EINVAL;
   1837	flush_dcache_page(page);
   1838	return 0;
   1839}
   1840
   1841static int insert_page_into_pte_locked(struct vm_area_struct *vma, pte_t *pte,
   1842			unsigned long addr, struct page *page, pgprot_t prot)
   1843{
   1844	if (!pte_none(*pte))
   1845		return -EBUSY;
   1846	/* Ok, finally just insert the thing.. */
   1847	get_page(page);
   1848	inc_mm_counter_fast(vma->vm_mm, mm_counter_file(page));
   1849	page_add_file_rmap(page, vma, false);
   1850	set_pte_at(vma->vm_mm, addr, pte, mk_pte(page, prot));
   1851	return 0;
   1852}
   1853
   1854/*
   1855 * This is the old fallback for page remapping.
   1856 *
   1857 * For historical reasons, it only allows reserved pages. Only
   1858 * old drivers should use this, and they needed to mark their
   1859 * pages reserved for the old functions anyway.
   1860 */
   1861static int insert_page(struct vm_area_struct *vma, unsigned long addr,
   1862			struct page *page, pgprot_t prot)
   1863{
   1864	int retval;
   1865	pte_t *pte;
   1866	spinlock_t *ptl;
   1867
   1868	retval = validate_page_before_insert(page);
   1869	if (retval)
   1870		goto out;
   1871	retval = -ENOMEM;
   1872	pte = get_locked_pte(vma->vm_mm, addr, &ptl);
   1873	if (!pte)
   1874		goto out;
   1875	retval = insert_page_into_pte_locked(vma, pte, addr, page, prot);
   1876	pte_unmap_unlock(pte, ptl);
   1877out:
   1878	return retval;
   1879}
   1880
   1881#ifdef pte_index
   1882static int insert_page_in_batch_locked(struct vm_area_struct *vma, pte_t *pte,
   1883			unsigned long addr, struct page *page, pgprot_t prot)
   1884{
   1885	int err;
   1886
   1887	if (!page_count(page))
   1888		return -EINVAL;
   1889	err = validate_page_before_insert(page);
   1890	if (err)
   1891		return err;
   1892	return insert_page_into_pte_locked(vma, pte, addr, page, prot);
   1893}
   1894
   1895/* insert_pages() amortizes the cost of spinlock operations
   1896 * when inserting pages in a loop. Arch *must* define pte_index.
   1897 */
   1898static int insert_pages(struct vm_area_struct *vma, unsigned long addr,
   1899			struct page **pages, unsigned long *num, pgprot_t prot)
   1900{
   1901	pmd_t *pmd = NULL;
   1902	pte_t *start_pte, *pte;
   1903	spinlock_t *pte_lock;
   1904	struct mm_struct *const mm = vma->vm_mm;
   1905	unsigned long curr_page_idx = 0;
   1906	unsigned long remaining_pages_total = *num;
   1907	unsigned long pages_to_write_in_pmd;
   1908	int ret;
   1909more:
   1910	ret = -EFAULT;
   1911	pmd = walk_to_pmd(mm, addr);
   1912	if (!pmd)
   1913		goto out;
   1914
   1915	pages_to_write_in_pmd = min_t(unsigned long,
   1916		remaining_pages_total, PTRS_PER_PTE - pte_index(addr));
   1917
   1918	/* Allocate the PTE if necessary; takes PMD lock once only. */
   1919	ret = -ENOMEM;
   1920	if (pte_alloc(mm, pmd))
   1921		goto out;
   1922
   1923	while (pages_to_write_in_pmd) {
   1924		int pte_idx = 0;
   1925		const int batch_size = min_t(int, pages_to_write_in_pmd, 8);
   1926
   1927		start_pte = pte_offset_map_lock(mm, pmd, addr, &pte_lock);
   1928		for (pte = start_pte; pte_idx < batch_size; ++pte, ++pte_idx) {
   1929			int err = insert_page_in_batch_locked(vma, pte,
   1930				addr, pages[curr_page_idx], prot);
   1931			if (unlikely(err)) {
   1932				pte_unmap_unlock(start_pte, pte_lock);
   1933				ret = err;
   1934				remaining_pages_total -= pte_idx;
   1935				goto out;
   1936			}
   1937			addr += PAGE_SIZE;
   1938			++curr_page_idx;
   1939		}
   1940		pte_unmap_unlock(start_pte, pte_lock);
   1941		pages_to_write_in_pmd -= batch_size;
   1942		remaining_pages_total -= batch_size;
   1943	}
   1944	if (remaining_pages_total)
   1945		goto more;
   1946	ret = 0;
   1947out:
   1948	*num = remaining_pages_total;
   1949	return ret;
   1950}
   1951#endif  /* ifdef pte_index */
   1952
   1953/**
   1954 * vm_insert_pages - insert multiple pages into user vma, batching the pmd lock.
   1955 * @vma: user vma to map to
   1956 * @addr: target start user address of these pages
   1957 * @pages: source kernel pages
   1958 * @num: in: number of pages to map. out: number of pages that were *not*
   1959 * mapped. (0 means all pages were successfully mapped).
   1960 *
   1961 * Preferred over vm_insert_page() when inserting multiple pages.
   1962 *
   1963 * In case of error, we may have mapped a subset of the provided
   1964 * pages. It is the caller's responsibility to account for this case.
   1965 *
   1966 * The same restrictions apply as in vm_insert_page().
   1967 */
   1968int vm_insert_pages(struct vm_area_struct *vma, unsigned long addr,
   1969			struct page **pages, unsigned long *num)
   1970{
   1971#ifdef pte_index
   1972	const unsigned long end_addr = addr + (*num * PAGE_SIZE) - 1;
   1973
   1974	if (addr < vma->vm_start || end_addr >= vma->vm_end)
   1975		return -EFAULT;
   1976	if (!(vma->vm_flags & VM_MIXEDMAP)) {
   1977		BUG_ON(mmap_read_trylock(vma->vm_mm));
   1978		BUG_ON(vma->vm_flags & VM_PFNMAP);
   1979		vma->vm_flags |= VM_MIXEDMAP;
   1980	}
   1981	/* Defer page refcount checking till we're about to map that page. */
   1982	return insert_pages(vma, addr, pages, num, vma->vm_page_prot);
   1983#else
   1984	unsigned long idx = 0, pgcount = *num;
   1985	int err = -EINVAL;
   1986
   1987	for (; idx < pgcount; ++idx) {
   1988		err = vm_insert_page(vma, addr + (PAGE_SIZE * idx), pages[idx]);
   1989		if (err)
   1990			break;
   1991	}
   1992	*num = pgcount - idx;
   1993	return err;
   1994#endif  /* ifdef pte_index */
   1995}
   1996EXPORT_SYMBOL(vm_insert_pages);
   1997
   1998/**
   1999 * vm_insert_page - insert single page into user vma
   2000 * @vma: user vma to map to
   2001 * @addr: target user address of this page
   2002 * @page: source kernel page
   2003 *
   2004 * This allows drivers to insert individual pages they've allocated
   2005 * into a user vma.
   2006 *
   2007 * The page has to be a nice clean _individual_ kernel allocation.
   2008 * If you allocate a compound page, you need to have marked it as
   2009 * such (__GFP_COMP), or manually just split the page up yourself
   2010 * (see split_page()).
   2011 *
   2012 * NOTE! Traditionally this was done with "remap_pfn_range()" which
   2013 * took an arbitrary page protection parameter. This doesn't allow
   2014 * that. Your vma protection will have to be set up correctly, which
   2015 * means that if you want a shared writable mapping, you'd better
   2016 * ask for a shared writable mapping!
   2017 *
   2018 * The page does not need to be reserved.
   2019 *
   2020 * Usually this function is called from f_op->mmap() handler
   2021 * under mm->mmap_lock write-lock, so it can change vma->vm_flags.
   2022 * Caller must set VM_MIXEDMAP on vma if it wants to call this
   2023 * function from other places, for example from page-fault handler.
   2024 *
   2025 * Return: %0 on success, negative error code otherwise.
   2026 */
   2027int vm_insert_page(struct vm_area_struct *vma, unsigned long addr,
   2028			struct page *page)
   2029{
   2030	if (addr < vma->vm_start || addr >= vma->vm_end)
   2031		return -EFAULT;
   2032	if (!page_count(page))
   2033		return -EINVAL;
   2034	if (!(vma->vm_flags & VM_MIXEDMAP)) {
   2035		BUG_ON(mmap_read_trylock(vma->vm_mm));
   2036		BUG_ON(vma->vm_flags & VM_PFNMAP);
   2037		vma->vm_flags |= VM_MIXEDMAP;
   2038	}
   2039	return insert_page(vma, addr, page, vma->vm_page_prot);
   2040}
   2041EXPORT_SYMBOL(vm_insert_page);
   2042
   2043/*
   2044 * __vm_map_pages - maps range of kernel pages into user vma
   2045 * @vma: user vma to map to
   2046 * @pages: pointer to array of source kernel pages
   2047 * @num: number of pages in page array
   2048 * @offset: user's requested vm_pgoff
   2049 *
   2050 * This allows drivers to map range of kernel pages into a user vma.
   2051 *
   2052 * Return: 0 on success and error code otherwise.
   2053 */
   2054static int __vm_map_pages(struct vm_area_struct *vma, struct page **pages,
   2055				unsigned long num, unsigned long offset)
   2056{
   2057	unsigned long count = vma_pages(vma);
   2058	unsigned long uaddr = vma->vm_start;
   2059	int ret, i;
   2060
   2061	/* Fail if the user requested offset is beyond the end of the object */
   2062	if (offset >= num)
   2063		return -ENXIO;
   2064
   2065	/* Fail if the user requested size exceeds available object size */
   2066	if (count > num - offset)
   2067		return -ENXIO;
   2068
   2069	for (i = 0; i < count; i++) {
   2070		ret = vm_insert_page(vma, uaddr, pages[offset + i]);
   2071		if (ret < 0)
   2072			return ret;
   2073		uaddr += PAGE_SIZE;
   2074	}
   2075
   2076	return 0;
   2077}
   2078
   2079/**
   2080 * vm_map_pages - maps range of kernel pages starts with non zero offset
   2081 * @vma: user vma to map to
   2082 * @pages: pointer to array of source kernel pages
   2083 * @num: number of pages in page array
   2084 *
   2085 * Maps an object consisting of @num pages, catering for the user's
   2086 * requested vm_pgoff
   2087 *
   2088 * If we fail to insert any page into the vma, the function will return
   2089 * immediately leaving any previously inserted pages present.  Callers
   2090 * from the mmap handler may immediately return the error as their caller
   2091 * will destroy the vma, removing any successfully inserted pages. Other
   2092 * callers should make their own arrangements for calling unmap_region().
   2093 *
   2094 * Context: Process context. Called by mmap handlers.
   2095 * Return: 0 on success and error code otherwise.
   2096 */
   2097int vm_map_pages(struct vm_area_struct *vma, struct page **pages,
   2098				unsigned long num)
   2099{
   2100	return __vm_map_pages(vma, pages, num, vma->vm_pgoff);
   2101}
   2102EXPORT_SYMBOL(vm_map_pages);
   2103
   2104/**
   2105 * vm_map_pages_zero - map range of kernel pages starts with zero offset
   2106 * @vma: user vma to map to
   2107 * @pages: pointer to array of source kernel pages
   2108 * @num: number of pages in page array
   2109 *
   2110 * Similar to vm_map_pages(), except that it explicitly sets the offset
   2111 * to 0. This function is intended for the drivers that did not consider
   2112 * vm_pgoff.
   2113 *
   2114 * Context: Process context. Called by mmap handlers.
   2115 * Return: 0 on success and error code otherwise.
   2116 */
   2117int vm_map_pages_zero(struct vm_area_struct *vma, struct page **pages,
   2118				unsigned long num)
   2119{
   2120	return __vm_map_pages(vma, pages, num, 0);
   2121}
   2122EXPORT_SYMBOL(vm_map_pages_zero);
   2123
   2124static vm_fault_t insert_pfn(struct vm_area_struct *vma, unsigned long addr,
   2125			pfn_t pfn, pgprot_t prot, bool mkwrite)
   2126{
   2127	struct mm_struct *mm = vma->vm_mm;
   2128	pte_t *pte, entry;
   2129	spinlock_t *ptl;
   2130
   2131	pte = get_locked_pte(mm, addr, &ptl);
   2132	if (!pte)
   2133		return VM_FAULT_OOM;
   2134	if (!pte_none(*pte)) {
   2135		if (mkwrite) {
   2136			/*
   2137			 * For read faults on private mappings the PFN passed
   2138			 * in may not match the PFN we have mapped if the
   2139			 * mapped PFN is a writeable COW page.  In the mkwrite
   2140			 * case we are creating a writable PTE for a shared
   2141			 * mapping and we expect the PFNs to match. If they
   2142			 * don't match, we are likely racing with block
   2143			 * allocation and mapping invalidation so just skip the
   2144			 * update.
   2145			 */
   2146			if (pte_pfn(*pte) != pfn_t_to_pfn(pfn)) {
   2147				WARN_ON_ONCE(!is_zero_pfn(pte_pfn(*pte)));
   2148				goto out_unlock;
   2149			}
   2150			entry = pte_mkyoung(*pte);
   2151			entry = maybe_mkwrite(pte_mkdirty(entry), vma);
   2152			if (ptep_set_access_flags(vma, addr, pte, entry, 1))
   2153				update_mmu_cache(vma, addr, pte);
   2154		}
   2155		goto out_unlock;
   2156	}
   2157
   2158	/* Ok, finally just insert the thing.. */
   2159	if (pfn_t_devmap(pfn))
   2160		entry = pte_mkdevmap(pfn_t_pte(pfn, prot));
   2161	else
   2162		entry = pte_mkspecial(pfn_t_pte(pfn, prot));
   2163
   2164	if (mkwrite) {
   2165		entry = pte_mkyoung(entry);
   2166		entry = maybe_mkwrite(pte_mkdirty(entry), vma);
   2167	}
   2168
   2169	set_pte_at(mm, addr, pte, entry);
   2170	update_mmu_cache(vma, addr, pte); /* XXX: why not for insert_page? */
   2171
   2172out_unlock:
   2173	pte_unmap_unlock(pte, ptl);
   2174	return VM_FAULT_NOPAGE;
   2175}
   2176
   2177/**
   2178 * vmf_insert_pfn_prot - insert single pfn into user vma with specified pgprot
   2179 * @vma: user vma to map to
   2180 * @addr: target user address of this page
   2181 * @pfn: source kernel pfn
   2182 * @pgprot: pgprot flags for the inserted page
   2183 *
   2184 * This is exactly like vmf_insert_pfn(), except that it allows drivers
   2185 * to override pgprot on a per-page basis.
   2186 *
   2187 * This only makes sense for IO mappings, and it makes no sense for
   2188 * COW mappings.  In general, using multiple vmas is preferable;
   2189 * vmf_insert_pfn_prot should only be used if using multiple VMAs is
   2190 * impractical.
   2191 *
   2192 * See vmf_insert_mixed_prot() for a discussion of the implication of using
   2193 * a value of @pgprot different from that of @vma->vm_page_prot.
   2194 *
   2195 * Context: Process context.  May allocate using %GFP_KERNEL.
   2196 * Return: vm_fault_t value.
   2197 */
   2198vm_fault_t vmf_insert_pfn_prot(struct vm_area_struct *vma, unsigned long addr,
   2199			unsigned long pfn, pgprot_t pgprot)
   2200{
   2201	/*
   2202	 * Technically, architectures with pte_special can avoid all these
   2203	 * restrictions (same for remap_pfn_range).  However we would like
   2204	 * consistency in testing and feature parity among all, so we should
   2205	 * try to keep these invariants in place for everybody.
   2206	 */
   2207	BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)));
   2208	BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) ==
   2209						(VM_PFNMAP|VM_MIXEDMAP));
   2210	BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags));
   2211	BUG_ON((vma->vm_flags & VM_MIXEDMAP) && pfn_valid(pfn));
   2212
   2213	if (addr < vma->vm_start || addr >= vma->vm_end)
   2214		return VM_FAULT_SIGBUS;
   2215
   2216	if (!pfn_modify_allowed(pfn, pgprot))
   2217		return VM_FAULT_SIGBUS;
   2218
   2219	track_pfn_insert(vma, &pgprot, __pfn_to_pfn_t(pfn, PFN_DEV));
   2220
   2221	return insert_pfn(vma, addr, __pfn_to_pfn_t(pfn, PFN_DEV), pgprot,
   2222			false);
   2223}
   2224EXPORT_SYMBOL(vmf_insert_pfn_prot);
   2225
   2226/**
   2227 * vmf_insert_pfn - insert single pfn into user vma
   2228 * @vma: user vma to map to
   2229 * @addr: target user address of this page
   2230 * @pfn: source kernel pfn
   2231 *
   2232 * Similar to vm_insert_page, this allows drivers to insert individual pages
   2233 * they've allocated into a user vma. Same comments apply.
   2234 *
   2235 * This function should only be called from a vm_ops->fault handler, and
   2236 * in that case the handler should return the result of this function.
   2237 *
   2238 * vma cannot be a COW mapping.
   2239 *
   2240 * As this is called only for pages that do not currently exist, we
   2241 * do not need to flush old virtual caches or the TLB.
   2242 *
   2243 * Context: Process context.  May allocate using %GFP_KERNEL.
   2244 * Return: vm_fault_t value.
   2245 */
   2246vm_fault_t vmf_insert_pfn(struct vm_area_struct *vma, unsigned long addr,
   2247			unsigned long pfn)
   2248{
   2249	return vmf_insert_pfn_prot(vma, addr, pfn, vma->vm_page_prot);
   2250}
   2251EXPORT_SYMBOL(vmf_insert_pfn);
   2252
   2253static bool vm_mixed_ok(struct vm_area_struct *vma, pfn_t pfn)
   2254{
   2255	/* these checks mirror the abort conditions in vm_normal_page */
   2256	if (vma->vm_flags & VM_MIXEDMAP)
   2257		return true;
   2258	if (pfn_t_devmap(pfn))
   2259		return true;
   2260	if (pfn_t_special(pfn))
   2261		return true;
   2262	if (is_zero_pfn(pfn_t_to_pfn(pfn)))
   2263		return true;
   2264	return false;
   2265}
   2266
   2267static vm_fault_t __vm_insert_mixed(struct vm_area_struct *vma,
   2268		unsigned long addr, pfn_t pfn, pgprot_t pgprot,
   2269		bool mkwrite)
   2270{
   2271	int err;
   2272
   2273	BUG_ON(!vm_mixed_ok(vma, pfn));
   2274
   2275	if (addr < vma->vm_start || addr >= vma->vm_end)
   2276		return VM_FAULT_SIGBUS;
   2277
   2278	track_pfn_insert(vma, &pgprot, pfn);
   2279
   2280	if (!pfn_modify_allowed(pfn_t_to_pfn(pfn), pgprot))
   2281		return VM_FAULT_SIGBUS;
   2282
   2283	/*
   2284	 * If we don't have pte special, then we have to use the pfn_valid()
   2285	 * based VM_MIXEDMAP scheme (see vm_normal_page), and thus we *must*
   2286	 * refcount the page if pfn_valid is true (hence insert_page rather
   2287	 * than insert_pfn).  If a zero_pfn were inserted into a VM_MIXEDMAP
   2288	 * without pte special, it would there be refcounted as a normal page.
   2289	 */
   2290	if (!IS_ENABLED(CONFIG_ARCH_HAS_PTE_SPECIAL) &&
   2291	    !pfn_t_devmap(pfn) && pfn_t_valid(pfn)) {
   2292		struct page *page;
   2293
   2294		/*
   2295		 * At this point we are committed to insert_page()
   2296		 * regardless of whether the caller specified flags that
   2297		 * result in pfn_t_has_page() == false.
   2298		 */
   2299		page = pfn_to_page(pfn_t_to_pfn(pfn));
   2300		err = insert_page(vma, addr, page, pgprot);
   2301	} else {
   2302		return insert_pfn(vma, addr, pfn, pgprot, mkwrite);
   2303	}
   2304
   2305	if (err == -ENOMEM)
   2306		return VM_FAULT_OOM;
   2307	if (err < 0 && err != -EBUSY)
   2308		return VM_FAULT_SIGBUS;
   2309
   2310	return VM_FAULT_NOPAGE;
   2311}
   2312
   2313/**
   2314 * vmf_insert_mixed_prot - insert single pfn into user vma with specified pgprot
   2315 * @vma: user vma to map to
   2316 * @addr: target user address of this page
   2317 * @pfn: source kernel pfn
   2318 * @pgprot: pgprot flags for the inserted page
   2319 *
   2320 * This is exactly like vmf_insert_mixed(), except that it allows drivers
   2321 * to override pgprot on a per-page basis.
   2322 *
   2323 * Typically this function should be used by drivers to set caching- and
   2324 * encryption bits different than those of @vma->vm_page_prot, because
   2325 * the caching- or encryption mode may not be known at mmap() time.
   2326 * This is ok as long as @vma->vm_page_prot is not used by the core vm
   2327 * to set caching and encryption bits for those vmas (except for COW pages).
   2328 * This is ensured by core vm only modifying these page table entries using
   2329 * functions that don't touch caching- or encryption bits, using pte_modify()
   2330 * if needed. (See for example mprotect()).
   2331 * Also when new page-table entries are created, this is only done using the
   2332 * fault() callback, and never using the value of vma->vm_page_prot,
   2333 * except for page-table entries that point to anonymous pages as the result
   2334 * of COW.
   2335 *
   2336 * Context: Process context.  May allocate using %GFP_KERNEL.
   2337 * Return: vm_fault_t value.
   2338 */
   2339vm_fault_t vmf_insert_mixed_prot(struct vm_area_struct *vma, unsigned long addr,
   2340				 pfn_t pfn, pgprot_t pgprot)
   2341{
   2342	return __vm_insert_mixed(vma, addr, pfn, pgprot, false);
   2343}
   2344EXPORT_SYMBOL(vmf_insert_mixed_prot);
   2345
   2346vm_fault_t vmf_insert_mixed(struct vm_area_struct *vma, unsigned long addr,
   2347		pfn_t pfn)
   2348{
   2349	return __vm_insert_mixed(vma, addr, pfn, vma->vm_page_prot, false);
   2350}
   2351EXPORT_SYMBOL(vmf_insert_mixed);
   2352
   2353/*
   2354 *  If the insertion of PTE failed because someone else already added a
   2355 *  different entry in the mean time, we treat that as success as we assume
   2356 *  the same entry was actually inserted.
   2357 */
   2358vm_fault_t vmf_insert_mixed_mkwrite(struct vm_area_struct *vma,
   2359		unsigned long addr, pfn_t pfn)
   2360{
   2361	return __vm_insert_mixed(vma, addr, pfn, vma->vm_page_prot, true);
   2362}
   2363EXPORT_SYMBOL(vmf_insert_mixed_mkwrite);
   2364
   2365/*
   2366 * maps a range of physical memory into the requested pages. the old
   2367 * mappings are removed. any references to nonexistent pages results
   2368 * in null mappings (currently treated as "copy-on-access")
   2369 */
   2370static int remap_pte_range(struct mm_struct *mm, pmd_t *pmd,
   2371			unsigned long addr, unsigned long end,
   2372			unsigned long pfn, pgprot_t prot)
   2373{
   2374	pte_t *pte, *mapped_pte;
   2375	spinlock_t *ptl;
   2376	int err = 0;
   2377
   2378	mapped_pte = pte = pte_alloc_map_lock(mm, pmd, addr, &ptl);
   2379	if (!pte)
   2380		return -ENOMEM;
   2381	arch_enter_lazy_mmu_mode();
   2382	do {
   2383		BUG_ON(!pte_none(*pte));
   2384		if (!pfn_modify_allowed(pfn, prot)) {
   2385			err = -EACCES;
   2386			break;
   2387		}
   2388		set_pte_at(mm, addr, pte, pte_mkspecial(pfn_pte(pfn, prot)));
   2389		pfn++;
   2390	} while (pte++, addr += PAGE_SIZE, addr != end);
   2391	arch_leave_lazy_mmu_mode();
   2392	pte_unmap_unlock(mapped_pte, ptl);
   2393	return err;
   2394}
   2395
   2396static inline int remap_pmd_range(struct mm_struct *mm, pud_t *pud,
   2397			unsigned long addr, unsigned long end,
   2398			unsigned long pfn, pgprot_t prot)
   2399{
   2400	pmd_t *pmd;
   2401	unsigned long next;
   2402	int err;
   2403
   2404	pfn -= addr >> PAGE_SHIFT;
   2405	pmd = pmd_alloc(mm, pud, addr);
   2406	if (!pmd)
   2407		return -ENOMEM;
   2408	VM_BUG_ON(pmd_trans_huge(*pmd));
   2409	do {
   2410		next = pmd_addr_end(addr, end);
   2411		err = remap_pte_range(mm, pmd, addr, next,
   2412				pfn + (addr >> PAGE_SHIFT), prot);
   2413		if (err)
   2414			return err;
   2415	} while (pmd++, addr = next, addr != end);
   2416	return 0;
   2417}
   2418
   2419static inline int remap_pud_range(struct mm_struct *mm, p4d_t *p4d,
   2420			unsigned long addr, unsigned long end,
   2421			unsigned long pfn, pgprot_t prot)
   2422{
   2423	pud_t *pud;
   2424	unsigned long next;
   2425	int err;
   2426
   2427	pfn -= addr >> PAGE_SHIFT;
   2428	pud = pud_alloc(mm, p4d, addr);
   2429	if (!pud)
   2430		return -ENOMEM;
   2431	do {
   2432		next = pud_addr_end(addr, end);
   2433		err = remap_pmd_range(mm, pud, addr, next,
   2434				pfn + (addr >> PAGE_SHIFT), prot);
   2435		if (err)
   2436			return err;
   2437	} while (pud++, addr = next, addr != end);
   2438	return 0;
   2439}
   2440
   2441static inline int remap_p4d_range(struct mm_struct *mm, pgd_t *pgd,
   2442			unsigned long addr, unsigned long end,
   2443			unsigned long pfn, pgprot_t prot)
   2444{
   2445	p4d_t *p4d;
   2446	unsigned long next;
   2447	int err;
   2448
   2449	pfn -= addr >> PAGE_SHIFT;
   2450	p4d = p4d_alloc(mm, pgd, addr);
   2451	if (!p4d)
   2452		return -ENOMEM;
   2453	do {
   2454		next = p4d_addr_end(addr, end);
   2455		err = remap_pud_range(mm, p4d, addr, next,
   2456				pfn + (addr >> PAGE_SHIFT), prot);
   2457		if (err)
   2458			return err;
   2459	} while (p4d++, addr = next, addr != end);
   2460	return 0;
   2461}
   2462
   2463/*
   2464 * Variant of remap_pfn_range that does not call track_pfn_remap.  The caller
   2465 * must have pre-validated the caching bits of the pgprot_t.
   2466 */
   2467int remap_pfn_range_notrack(struct vm_area_struct *vma, unsigned long addr,
   2468		unsigned long pfn, unsigned long size, pgprot_t prot)
   2469{
   2470	pgd_t *pgd;
   2471	unsigned long next;
   2472	unsigned long end = addr + PAGE_ALIGN(size);
   2473	struct mm_struct *mm = vma->vm_mm;
   2474	int err;
   2475
   2476	if (WARN_ON_ONCE(!PAGE_ALIGNED(addr)))
   2477		return -EINVAL;
   2478
   2479	/*
   2480	 * Physically remapped pages are special. Tell the
   2481	 * rest of the world about it:
   2482	 *   VM_IO tells people not to look at these pages
   2483	 *	(accesses can have side effects).
   2484	 *   VM_PFNMAP tells the core MM that the base pages are just
   2485	 *	raw PFN mappings, and do not have a "struct page" associated
   2486	 *	with them.
   2487	 *   VM_DONTEXPAND
   2488	 *      Disable vma merging and expanding with mremap().
   2489	 *   VM_DONTDUMP
   2490	 *      Omit vma from core dump, even when VM_IO turned off.
   2491	 *
   2492	 * There's a horrible special case to handle copy-on-write
   2493	 * behaviour that some programs depend on. We mark the "original"
   2494	 * un-COW'ed pages by matching them up with "vma->vm_pgoff".
   2495	 * See vm_normal_page() for details.
   2496	 */
   2497	if (is_cow_mapping(vma->vm_flags)) {
   2498		if (addr != vma->vm_start || end != vma->vm_end)
   2499			return -EINVAL;
   2500		vma->vm_pgoff = pfn;
   2501	}
   2502
   2503	vma->vm_flags |= VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP;
   2504
   2505	BUG_ON(addr >= end);
   2506	pfn -= addr >> PAGE_SHIFT;
   2507	pgd = pgd_offset(mm, addr);
   2508	flush_cache_range(vma, addr, end);
   2509	do {
   2510		next = pgd_addr_end(addr, end);
   2511		err = remap_p4d_range(mm, pgd, addr, next,
   2512				pfn + (addr >> PAGE_SHIFT), prot);
   2513		if (err)
   2514			return err;
   2515	} while (pgd++, addr = next, addr != end);
   2516
   2517	return 0;
   2518}
   2519
   2520/**
   2521 * remap_pfn_range - remap kernel memory to userspace
   2522 * @vma: user vma to map to
   2523 * @addr: target page aligned user address to start at
   2524 * @pfn: page frame number of kernel physical memory address
   2525 * @size: size of mapping area
   2526 * @prot: page protection flags for this mapping
   2527 *
   2528 * Note: this is only safe if the mm semaphore is held when called.
   2529 *
   2530 * Return: %0 on success, negative error code otherwise.
   2531 */
   2532int remap_pfn_range(struct vm_area_struct *vma, unsigned long addr,
   2533		    unsigned long pfn, unsigned long size, pgprot_t prot)
   2534{
   2535	int err;
   2536
   2537	err = track_pfn_remap(vma, &prot, pfn, addr, PAGE_ALIGN(size));
   2538	if (err)
   2539		return -EINVAL;
   2540
   2541	err = remap_pfn_range_notrack(vma, addr, pfn, size, prot);
   2542	if (err)
   2543		untrack_pfn(vma, pfn, PAGE_ALIGN(size));
   2544	return err;
   2545}
   2546EXPORT_SYMBOL(remap_pfn_range);
   2547
   2548/**
   2549 * vm_iomap_memory - remap memory to userspace
   2550 * @vma: user vma to map to
   2551 * @start: start of the physical memory to be mapped
   2552 * @len: size of area
   2553 *
   2554 * This is a simplified io_remap_pfn_range() for common driver use. The
   2555 * driver just needs to give us the physical memory range to be mapped,
   2556 * we'll figure out the rest from the vma information.
   2557 *
   2558 * NOTE! Some drivers might want to tweak vma->vm_page_prot first to get
   2559 * whatever write-combining details or similar.
   2560 *
   2561 * Return: %0 on success, negative error code otherwise.
   2562 */
   2563int vm_iomap_memory(struct vm_area_struct *vma, phys_addr_t start, unsigned long len)
   2564{
   2565	unsigned long vm_len, pfn, pages;
   2566
   2567	/* Check that the physical memory area passed in looks valid */
   2568	if (start + len < start)
   2569		return -EINVAL;
   2570	/*
   2571	 * You *really* shouldn't map things that aren't page-aligned,
   2572	 * but we've historically allowed it because IO memory might
   2573	 * just have smaller alignment.
   2574	 */
   2575	len += start & ~PAGE_MASK;
   2576	pfn = start >> PAGE_SHIFT;
   2577	pages = (len + ~PAGE_MASK) >> PAGE_SHIFT;
   2578	if (pfn + pages < pfn)
   2579		return -EINVAL;
   2580
   2581	/* We start the mapping 'vm_pgoff' pages into the area */
   2582	if (vma->vm_pgoff > pages)
   2583		return -EINVAL;
   2584	pfn += vma->vm_pgoff;
   2585	pages -= vma->vm_pgoff;
   2586
   2587	/* Can we fit all of the mapping? */
   2588	vm_len = vma->vm_end - vma->vm_start;
   2589	if (vm_len >> PAGE_SHIFT > pages)
   2590		return -EINVAL;
   2591
   2592	/* Ok, let it rip */
   2593	return io_remap_pfn_range(vma, vma->vm_start, pfn, vm_len, vma->vm_page_prot);
   2594}
   2595EXPORT_SYMBOL(vm_iomap_memory);
   2596
   2597static int apply_to_pte_range(struct mm_struct *mm, pmd_t *pmd,
   2598				     unsigned long addr, unsigned long end,
   2599				     pte_fn_t fn, void *data, bool create,
   2600				     pgtbl_mod_mask *mask)
   2601{
   2602	pte_t *pte, *mapped_pte;
   2603	int err = 0;
   2604	spinlock_t *ptl;
   2605
   2606	if (create) {
   2607		mapped_pte = pte = (mm == &init_mm) ?
   2608			pte_alloc_kernel_track(pmd, addr, mask) :
   2609			pte_alloc_map_lock(mm, pmd, addr, &ptl);
   2610		if (!pte)
   2611			return -ENOMEM;
   2612	} else {
   2613		mapped_pte = pte = (mm == &init_mm) ?
   2614			pte_offset_kernel(pmd, addr) :
   2615			pte_offset_map_lock(mm, pmd, addr, &ptl);
   2616	}
   2617
   2618	BUG_ON(pmd_huge(*pmd));
   2619
   2620	arch_enter_lazy_mmu_mode();
   2621
   2622	if (fn) {
   2623		do {
   2624			if (create || !pte_none(*pte)) {
   2625				err = fn(pte++, addr, data);
   2626				if (err)
   2627					break;
   2628			}
   2629		} while (addr += PAGE_SIZE, addr != end);
   2630	}
   2631	*mask |= PGTBL_PTE_MODIFIED;
   2632
   2633	arch_leave_lazy_mmu_mode();
   2634
   2635	if (mm != &init_mm)
   2636		pte_unmap_unlock(mapped_pte, ptl);
   2637	return err;
   2638}
   2639
   2640static int apply_to_pmd_range(struct mm_struct *mm, pud_t *pud,
   2641				     unsigned long addr, unsigned long end,
   2642				     pte_fn_t fn, void *data, bool create,
   2643				     pgtbl_mod_mask *mask)
   2644{
   2645	pmd_t *pmd;
   2646	unsigned long next;
   2647	int err = 0;
   2648
   2649	BUG_ON(pud_huge(*pud));
   2650
   2651	if (create) {
   2652		pmd = pmd_alloc_track(mm, pud, addr, mask);
   2653		if (!pmd)
   2654			return -ENOMEM;
   2655	} else {
   2656		pmd = pmd_offset(pud, addr);
   2657	}
   2658	do {
   2659		next = pmd_addr_end(addr, end);
   2660		if (pmd_none(*pmd) && !create)
   2661			continue;
   2662		if (WARN_ON_ONCE(pmd_leaf(*pmd)))
   2663			return -EINVAL;
   2664		if (!pmd_none(*pmd) && WARN_ON_ONCE(pmd_bad(*pmd))) {
   2665			if (!create)
   2666				continue;
   2667			pmd_clear_bad(pmd);
   2668		}
   2669		err = apply_to_pte_range(mm, pmd, addr, next,
   2670					 fn, data, create, mask);
   2671		if (err)
   2672			break;
   2673	} while (pmd++, addr = next, addr != end);
   2674
   2675	return err;
   2676}
   2677
   2678static int apply_to_pud_range(struct mm_struct *mm, p4d_t *p4d,
   2679				     unsigned long addr, unsigned long end,
   2680				     pte_fn_t fn, void *data, bool create,
   2681				     pgtbl_mod_mask *mask)
   2682{
   2683	pud_t *pud;
   2684	unsigned long next;
   2685	int err = 0;
   2686
   2687	if (create) {
   2688		pud = pud_alloc_track(mm, p4d, addr, mask);
   2689		if (!pud)
   2690			return -ENOMEM;
   2691	} else {
   2692		pud = pud_offset(p4d, addr);
   2693	}
   2694	do {
   2695		next = pud_addr_end(addr, end);
   2696		if (pud_none(*pud) && !create)
   2697			continue;
   2698		if (WARN_ON_ONCE(pud_leaf(*pud)))
   2699			return -EINVAL;
   2700		if (!pud_none(*pud) && WARN_ON_ONCE(pud_bad(*pud))) {
   2701			if (!create)
   2702				continue;
   2703			pud_clear_bad(pud);
   2704		}
   2705		err = apply_to_pmd_range(mm, pud, addr, next,
   2706					 fn, data, create, mask);
   2707		if (err)
   2708			break;
   2709	} while (pud++, addr = next, addr != end);
   2710
   2711	return err;
   2712}
   2713
   2714static int apply_to_p4d_range(struct mm_struct *mm, pgd_t *pgd,
   2715				     unsigned long addr, unsigned long end,
   2716				     pte_fn_t fn, void *data, bool create,
   2717				     pgtbl_mod_mask *mask)
   2718{
   2719	p4d_t *p4d;
   2720	unsigned long next;
   2721	int err = 0;
   2722
   2723	if (create) {
   2724		p4d = p4d_alloc_track(mm, pgd, addr, mask);
   2725		if (!p4d)
   2726			return -ENOMEM;
   2727	} else {
   2728		p4d = p4d_offset(pgd, addr);
   2729	}
   2730	do {
   2731		next = p4d_addr_end(addr, end);
   2732		if (p4d_none(*p4d) && !create)
   2733			continue;
   2734		if (WARN_ON_ONCE(p4d_leaf(*p4d)))
   2735			return -EINVAL;
   2736		if (!p4d_none(*p4d) && WARN_ON_ONCE(p4d_bad(*p4d))) {
   2737			if (!create)
   2738				continue;
   2739			p4d_clear_bad(p4d);
   2740		}
   2741		err = apply_to_pud_range(mm, p4d, addr, next,
   2742					 fn, data, create, mask);
   2743		if (err)
   2744			break;
   2745	} while (p4d++, addr = next, addr != end);
   2746
   2747	return err;
   2748}
   2749
   2750static int __apply_to_page_range(struct mm_struct *mm, unsigned long addr,
   2751				 unsigned long size, pte_fn_t fn,
   2752				 void *data, bool create)
   2753{
   2754	pgd_t *pgd;
   2755	unsigned long start = addr, next;
   2756	unsigned long end = addr + size;
   2757	pgtbl_mod_mask mask = 0;
   2758	int err = 0;
   2759
   2760	if (WARN_ON(addr >= end))
   2761		return -EINVAL;
   2762
   2763	pgd = pgd_offset(mm, addr);
   2764	do {
   2765		next = pgd_addr_end(addr, end);
   2766		if (pgd_none(*pgd) && !create)
   2767			continue;
   2768		if (WARN_ON_ONCE(pgd_leaf(*pgd)))
   2769			return -EINVAL;
   2770		if (!pgd_none(*pgd) && WARN_ON_ONCE(pgd_bad(*pgd))) {
   2771			if (!create)
   2772				continue;
   2773			pgd_clear_bad(pgd);
   2774		}
   2775		err = apply_to_p4d_range(mm, pgd, addr, next,
   2776					 fn, data, create, &mask);
   2777		if (err)
   2778			break;
   2779	} while (pgd++, addr = next, addr != end);
   2780
   2781	if (mask & ARCH_PAGE_TABLE_SYNC_MASK)
   2782		arch_sync_kernel_mappings(start, start + size);
   2783
   2784	return err;
   2785}
   2786
   2787/*
   2788 * Scan a region of virtual memory, filling in page tables as necessary
   2789 * and calling a provided function on each leaf page table.
   2790 */
   2791int apply_to_page_range(struct mm_struct *mm, unsigned long addr,
   2792			unsigned long size, pte_fn_t fn, void *data)
   2793{
   2794	return __apply_to_page_range(mm, addr, size, fn, data, true);
   2795}
   2796EXPORT_SYMBOL_GPL(apply_to_page_range);
   2797
   2798/*
   2799 * Scan a region of virtual memory, calling a provided function on
   2800 * each leaf page table where it exists.
   2801 *
   2802 * Unlike apply_to_page_range, this does _not_ fill in page tables
   2803 * where they are absent.
   2804 */
   2805int apply_to_existing_page_range(struct mm_struct *mm, unsigned long addr,
   2806				 unsigned long size, pte_fn_t fn, void *data)
   2807{
   2808	return __apply_to_page_range(mm, addr, size, fn, data, false);
   2809}
   2810EXPORT_SYMBOL_GPL(apply_to_existing_page_range);
   2811
   2812/*
   2813 * handle_pte_fault chooses page fault handler according to an entry which was
   2814 * read non-atomically.  Before making any commitment, on those architectures
   2815 * or configurations (e.g. i386 with PAE) which might give a mix of unmatched
   2816 * parts, do_swap_page must check under lock before unmapping the pte and
   2817 * proceeding (but do_wp_page is only called after already making such a check;
   2818 * and do_anonymous_page can safely check later on).
   2819 */
   2820static inline int pte_unmap_same(struct vm_fault *vmf)
   2821{
   2822	int same = 1;
   2823#if defined(CONFIG_SMP) || defined(CONFIG_PREEMPTION)
   2824	if (sizeof(pte_t) > sizeof(unsigned long)) {
   2825		spinlock_t *ptl = pte_lockptr(vmf->vma->vm_mm, vmf->pmd);
   2826		spin_lock(ptl);
   2827		same = pte_same(*vmf->pte, vmf->orig_pte);
   2828		spin_unlock(ptl);
   2829	}
   2830#endif
   2831	pte_unmap(vmf->pte);
   2832	vmf->pte = NULL;
   2833	return same;
   2834}
   2835
   2836static inline bool __wp_page_copy_user(struct page *dst, struct page *src,
   2837				       struct vm_fault *vmf)
   2838{
   2839	bool ret;
   2840	void *kaddr;
   2841	void __user *uaddr;
   2842	bool locked = false;
   2843	struct vm_area_struct *vma = vmf->vma;
   2844	struct mm_struct *mm = vma->vm_mm;
   2845	unsigned long addr = vmf->address;
   2846
   2847	if (likely(src)) {
   2848		copy_user_highpage(dst, src, addr, vma);
   2849		return true;
   2850	}
   2851
   2852	/*
   2853	 * If the source page was a PFN mapping, we don't have
   2854	 * a "struct page" for it. We do a best-effort copy by
   2855	 * just copying from the original user address. If that
   2856	 * fails, we just zero-fill it. Live with it.
   2857	 */
   2858	kaddr = kmap_atomic(dst);
   2859	uaddr = (void __user *)(addr & PAGE_MASK);
   2860
   2861	/*
   2862	 * On architectures with software "accessed" bits, we would
   2863	 * take a double page fault, so mark it accessed here.
   2864	 */
   2865	if (arch_faults_on_old_pte() && !pte_young(vmf->orig_pte)) {
   2866		pte_t entry;
   2867
   2868		vmf->pte = pte_offset_map_lock(mm, vmf->pmd, addr, &vmf->ptl);
   2869		locked = true;
   2870		if (!likely(pte_same(*vmf->pte, vmf->orig_pte))) {
   2871			/*
   2872			 * Other thread has already handled the fault
   2873			 * and update local tlb only
   2874			 */
   2875			update_mmu_tlb(vma, addr, vmf->pte);
   2876			ret = false;
   2877			goto pte_unlock;
   2878		}
   2879
   2880		entry = pte_mkyoung(vmf->orig_pte);
   2881		if (ptep_set_access_flags(vma, addr, vmf->pte, entry, 0))
   2882			update_mmu_cache(vma, addr, vmf->pte);
   2883	}
   2884
   2885	/*
   2886	 * This really shouldn't fail, because the page is there
   2887	 * in the page tables. But it might just be unreadable,
   2888	 * in which case we just give up and fill the result with
   2889	 * zeroes.
   2890	 */
   2891	if (__copy_from_user_inatomic(kaddr, uaddr, PAGE_SIZE)) {
   2892		if (locked)
   2893			goto warn;
   2894
   2895		/* Re-validate under PTL if the page is still mapped */
   2896		vmf->pte = pte_offset_map_lock(mm, vmf->pmd, addr, &vmf->ptl);
   2897		locked = true;
   2898		if (!likely(pte_same(*vmf->pte, vmf->orig_pte))) {
   2899			/* The PTE changed under us, update local tlb */
   2900			update_mmu_tlb(vma, addr, vmf->pte);
   2901			ret = false;
   2902			goto pte_unlock;
   2903		}
   2904
   2905		/*
   2906		 * The same page can be mapped back since last copy attempt.
   2907		 * Try to copy again under PTL.
   2908		 */
   2909		if (__copy_from_user_inatomic(kaddr, uaddr, PAGE_SIZE)) {
   2910			/*
   2911			 * Give a warn in case there can be some obscure
   2912			 * use-case
   2913			 */
   2914warn:
   2915			WARN_ON_ONCE(1);
   2916			clear_page(kaddr);
   2917		}
   2918	}
   2919
   2920	ret = true;
   2921
   2922pte_unlock:
   2923	if (locked)
   2924		pte_unmap_unlock(vmf->pte, vmf->ptl);
   2925	kunmap_atomic(kaddr);
   2926	flush_dcache_page(dst);
   2927
   2928	return ret;
   2929}
   2930
   2931static gfp_t __get_fault_gfp_mask(struct vm_area_struct *vma)
   2932{
   2933	struct file *vm_file = vma->vm_file;
   2934
   2935	if (vm_file)
   2936		return mapping_gfp_mask(vm_file->f_mapping) | __GFP_FS | __GFP_IO;
   2937
   2938	/*
   2939	 * Special mappings (e.g. VDSO) do not have any file so fake
   2940	 * a default GFP_KERNEL for them.
   2941	 */
   2942	return GFP_KERNEL;
   2943}
   2944
   2945/*
   2946 * Notify the address space that the page is about to become writable so that
   2947 * it can prohibit this or wait for the page to get into an appropriate state.
   2948 *
   2949 * We do this without the lock held, so that it can sleep if it needs to.
   2950 */
   2951static vm_fault_t do_page_mkwrite(struct vm_fault *vmf)
   2952{
   2953	vm_fault_t ret;
   2954	struct page *page = vmf->page;
   2955	unsigned int old_flags = vmf->flags;
   2956
   2957	vmf->flags = FAULT_FLAG_WRITE|FAULT_FLAG_MKWRITE;
   2958
   2959	if (vmf->vma->vm_file &&
   2960	    IS_SWAPFILE(vmf->vma->vm_file->f_mapping->host))
   2961		return VM_FAULT_SIGBUS;
   2962
   2963	ret = vmf->vma->vm_ops->page_mkwrite(vmf);
   2964	/* Restore original flags so that caller is not surprised */
   2965	vmf->flags = old_flags;
   2966	if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE)))
   2967		return ret;
   2968	if (unlikely(!(ret & VM_FAULT_LOCKED))) {
   2969		lock_page(page);
   2970		if (!page->mapping) {
   2971			unlock_page(page);
   2972			return 0; /* retry */
   2973		}
   2974		ret |= VM_FAULT_LOCKED;
   2975	} else
   2976		VM_BUG_ON_PAGE(!PageLocked(page), page);
   2977	return ret;
   2978}
   2979
   2980/*
   2981 * Handle dirtying of a page in shared file mapping on a write fault.
   2982 *
   2983 * The function expects the page to be locked and unlocks it.
   2984 */
   2985static vm_fault_t fault_dirty_shared_page(struct vm_fault *vmf)
   2986{
   2987	struct vm_area_struct *vma = vmf->vma;
   2988	struct address_space *mapping;
   2989	struct page *page = vmf->page;
   2990	bool dirtied;
   2991	bool page_mkwrite = vma->vm_ops && vma->vm_ops->page_mkwrite;
   2992
   2993	dirtied = set_page_dirty(page);
   2994	VM_BUG_ON_PAGE(PageAnon(page), page);
   2995	/*
   2996	 * Take a local copy of the address_space - page.mapping may be zeroed
   2997	 * by truncate after unlock_page().   The address_space itself remains
   2998	 * pinned by vma->vm_file's reference.  We rely on unlock_page()'s
   2999	 * release semantics to prevent the compiler from undoing this copying.
   3000	 */
   3001	mapping = page_rmapping(page);
   3002	unlock_page(page);
   3003
   3004	if (!page_mkwrite)
   3005		file_update_time(vma->vm_file);
   3006
   3007	/*
   3008	 * Throttle page dirtying rate down to writeback speed.
   3009	 *
   3010	 * mapping may be NULL here because some device drivers do not
   3011	 * set page.mapping but still dirty their pages
   3012	 *
   3013	 * Drop the mmap_lock before waiting on IO, if we can. The file
   3014	 * is pinning the mapping, as per above.
   3015	 */
   3016	if ((dirtied || page_mkwrite) && mapping) {
   3017		struct file *fpin;
   3018
   3019		fpin = maybe_unlock_mmap_for_io(vmf, NULL);
   3020		balance_dirty_pages_ratelimited(mapping);
   3021		if (fpin) {
   3022			fput(fpin);
   3023			return VM_FAULT_RETRY;
   3024		}
   3025	}
   3026
   3027	return 0;
   3028}
   3029
   3030/*
   3031 * Handle write page faults for pages that can be reused in the current vma
   3032 *
   3033 * This can happen either due to the mapping being with the VM_SHARED flag,
   3034 * or due to us being the last reference standing to the page. In either
   3035 * case, all we need to do here is to mark the page as writable and update
   3036 * any related book-keeping.
   3037 */
   3038static inline void wp_page_reuse(struct vm_fault *vmf)
   3039	__releases(vmf->ptl)
   3040{
   3041	struct vm_area_struct *vma = vmf->vma;
   3042	struct page *page = vmf->page;
   3043	pte_t entry;
   3044
   3045	VM_BUG_ON(!(vmf->flags & FAULT_FLAG_WRITE));
   3046	VM_BUG_ON(PageAnon(page) && !PageAnonExclusive(page));
   3047
   3048	/*
   3049	 * Clear the pages cpupid information as the existing
   3050	 * information potentially belongs to a now completely
   3051	 * unrelated process.
   3052	 */
   3053	if (page)
   3054		page_cpupid_xchg_last(page, (1 << LAST_CPUPID_SHIFT) - 1);
   3055
   3056	flush_cache_page(vma, vmf->address, pte_pfn(vmf->orig_pte));
   3057	entry = pte_mkyoung(vmf->orig_pte);
   3058	entry = maybe_mkwrite(pte_mkdirty(entry), vma);
   3059	if (ptep_set_access_flags(vma, vmf->address, vmf->pte, entry, 1))
   3060		update_mmu_cache(vma, vmf->address, vmf->pte);
   3061	pte_unmap_unlock(vmf->pte, vmf->ptl);
   3062	count_vm_event(PGREUSE);
   3063}
   3064
   3065/*
   3066 * Handle the case of a page which we actually need to copy to a new page,
   3067 * either due to COW or unsharing.
   3068 *
   3069 * Called with mmap_lock locked and the old page referenced, but
   3070 * without the ptl held.
   3071 *
   3072 * High level logic flow:
   3073 *
   3074 * - Allocate a page, copy the content of the old page to the new one.
   3075 * - Handle book keeping and accounting - cgroups, mmu-notifiers, etc.
   3076 * - Take the PTL. If the pte changed, bail out and release the allocated page
   3077 * - If the pte is still the way we remember it, update the page table and all
   3078 *   relevant references. This includes dropping the reference the page-table
   3079 *   held to the old page, as well as updating the rmap.
   3080 * - In any case, unlock the PTL and drop the reference we took to the old page.
   3081 */
   3082static vm_fault_t wp_page_copy(struct vm_fault *vmf)
   3083{
   3084	const bool unshare = vmf->flags & FAULT_FLAG_UNSHARE;
   3085	struct vm_area_struct *vma = vmf->vma;
   3086	struct mm_struct *mm = vma->vm_mm;
   3087	struct page *old_page = vmf->page;
   3088	struct page *new_page = NULL;
   3089	pte_t entry;
   3090	int page_copied = 0;
   3091	struct mmu_notifier_range range;
   3092
   3093	delayacct_wpcopy_start();
   3094
   3095	if (unlikely(anon_vma_prepare(vma)))
   3096		goto oom;
   3097
   3098	if (is_zero_pfn(pte_pfn(vmf->orig_pte))) {
   3099		new_page = alloc_zeroed_user_highpage_movable(vma,
   3100							      vmf->address);
   3101		if (!new_page)
   3102			goto oom;
   3103	} else {
   3104		new_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma,
   3105				vmf->address);
   3106		if (!new_page)
   3107			goto oom;
   3108
   3109		if (!__wp_page_copy_user(new_page, old_page, vmf)) {
   3110			/*
   3111			 * COW failed, if the fault was solved by other,
   3112			 * it's fine. If not, userspace would re-fault on
   3113			 * the same address and we will handle the fault
   3114			 * from the second attempt.
   3115			 */
   3116			put_page(new_page);
   3117			if (old_page)
   3118				put_page(old_page);
   3119
   3120			delayacct_wpcopy_end();
   3121			return 0;
   3122		}
   3123	}
   3124
   3125	if (mem_cgroup_charge(page_folio(new_page), mm, GFP_KERNEL))
   3126		goto oom_free_new;
   3127	cgroup_throttle_swaprate(new_page, GFP_KERNEL);
   3128
   3129	__SetPageUptodate(new_page);
   3130
   3131	mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, mm,
   3132				vmf->address & PAGE_MASK,
   3133				(vmf->address & PAGE_MASK) + PAGE_SIZE);
   3134	mmu_notifier_invalidate_range_start(&range);
   3135
   3136	/*
   3137	 * Re-check the pte - we dropped the lock
   3138	 */
   3139	vmf->pte = pte_offset_map_lock(mm, vmf->pmd, vmf->address, &vmf->ptl);
   3140	if (likely(pte_same(*vmf->pte, vmf->orig_pte))) {
   3141		if (old_page) {
   3142			if (!PageAnon(old_page)) {
   3143				dec_mm_counter_fast(mm,
   3144						mm_counter_file(old_page));
   3145				inc_mm_counter_fast(mm, MM_ANONPAGES);
   3146			}
   3147		} else {
   3148			inc_mm_counter_fast(mm, MM_ANONPAGES);
   3149		}
   3150		flush_cache_page(vma, vmf->address, pte_pfn(vmf->orig_pte));
   3151		entry = mk_pte(new_page, vma->vm_page_prot);
   3152		entry = pte_sw_mkyoung(entry);
   3153		if (unlikely(unshare)) {
   3154			if (pte_soft_dirty(vmf->orig_pte))
   3155				entry = pte_mksoft_dirty(entry);
   3156			if (pte_uffd_wp(vmf->orig_pte))
   3157				entry = pte_mkuffd_wp(entry);
   3158		} else {
   3159			entry = maybe_mkwrite(pte_mkdirty(entry), vma);
   3160		}
   3161
   3162		/*
   3163		 * Clear the pte entry and flush it first, before updating the
   3164		 * pte with the new entry, to keep TLBs on different CPUs in
   3165		 * sync. This code used to set the new PTE then flush TLBs, but
   3166		 * that left a window where the new PTE could be loaded into
   3167		 * some TLBs while the old PTE remains in others.
   3168		 */
   3169		ptep_clear_flush_notify(vma, vmf->address, vmf->pte);
   3170		page_add_new_anon_rmap(new_page, vma, vmf->address);
   3171		lru_cache_add_inactive_or_unevictable(new_page, vma);
   3172		/*
   3173		 * We call the notify macro here because, when using secondary
   3174		 * mmu page tables (such as kvm shadow page tables), we want the
   3175		 * new page to be mapped directly into the secondary page table.
   3176		 */
   3177		BUG_ON(unshare && pte_write(entry));
   3178		set_pte_at_notify(mm, vmf->address, vmf->pte, entry);
   3179		update_mmu_cache(vma, vmf->address, vmf->pte);
   3180		if (old_page) {
   3181			/*
   3182			 * Only after switching the pte to the new page may
   3183			 * we remove the mapcount here. Otherwise another
   3184			 * process may come and find the rmap count decremented
   3185			 * before the pte is switched to the new page, and
   3186			 * "reuse" the old page writing into it while our pte
   3187			 * here still points into it and can be read by other
   3188			 * threads.
   3189			 *
   3190			 * The critical issue is to order this
   3191			 * page_remove_rmap with the ptp_clear_flush above.
   3192			 * Those stores are ordered by (if nothing else,)
   3193			 * the barrier present in the atomic_add_negative
   3194			 * in page_remove_rmap.
   3195			 *
   3196			 * Then the TLB flush in ptep_clear_flush ensures that
   3197			 * no process can access the old page before the
   3198			 * decremented mapcount is visible. And the old page
   3199			 * cannot be reused until after the decremented
   3200			 * mapcount is visible. So transitively, TLBs to
   3201			 * old page will be flushed before it can be reused.
   3202			 */
   3203			page_remove_rmap(old_page, vma, false);
   3204		}
   3205
   3206		/* Free the old page.. */
   3207		new_page = old_page;
   3208		page_copied = 1;
   3209	} else {
   3210		update_mmu_tlb(vma, vmf->address, vmf->pte);
   3211	}
   3212
   3213	if (new_page)
   3214		put_page(new_page);
   3215
   3216	pte_unmap_unlock(vmf->pte, vmf->ptl);
   3217	/*
   3218	 * No need to double call mmu_notifier->invalidate_range() callback as
   3219	 * the above ptep_clear_flush_notify() did already call it.
   3220	 */
   3221	mmu_notifier_invalidate_range_only_end(&range);
   3222	if (old_page) {
   3223		if (page_copied)
   3224			free_swap_cache(old_page);
   3225		put_page(old_page);
   3226	}
   3227
   3228	delayacct_wpcopy_end();
   3229	return (page_copied && !unshare) ? VM_FAULT_WRITE : 0;
   3230oom_free_new:
   3231	put_page(new_page);
   3232oom:
   3233	if (old_page)
   3234		put_page(old_page);
   3235
   3236	delayacct_wpcopy_end();
   3237	return VM_FAULT_OOM;
   3238}
   3239
   3240/**
   3241 * finish_mkwrite_fault - finish page fault for a shared mapping, making PTE
   3242 *			  writeable once the page is prepared
   3243 *
   3244 * @vmf: structure describing the fault
   3245 *
   3246 * This function handles all that is needed to finish a write page fault in a
   3247 * shared mapping due to PTE being read-only once the mapped page is prepared.
   3248 * It handles locking of PTE and modifying it.
   3249 *
   3250 * The function expects the page to be locked or other protection against
   3251 * concurrent faults / writeback (such as DAX radix tree locks).
   3252 *
   3253 * Return: %0 on success, %VM_FAULT_NOPAGE when PTE got changed before
   3254 * we acquired PTE lock.
   3255 */
   3256vm_fault_t finish_mkwrite_fault(struct vm_fault *vmf)
   3257{
   3258	WARN_ON_ONCE(!(vmf->vma->vm_flags & VM_SHARED));
   3259	vmf->pte = pte_offset_map_lock(vmf->vma->vm_mm, vmf->pmd, vmf->address,
   3260				       &vmf->ptl);
   3261	/*
   3262	 * We might have raced with another page fault while we released the
   3263	 * pte_offset_map_lock.
   3264	 */
   3265	if (!pte_same(*vmf->pte, vmf->orig_pte)) {
   3266		update_mmu_tlb(vmf->vma, vmf->address, vmf->pte);
   3267		pte_unmap_unlock(vmf->pte, vmf->ptl);
   3268		return VM_FAULT_NOPAGE;
   3269	}
   3270	wp_page_reuse(vmf);
   3271	return 0;
   3272}
   3273
   3274/*
   3275 * Handle write page faults for VM_MIXEDMAP or VM_PFNMAP for a VM_SHARED
   3276 * mapping
   3277 */
   3278static vm_fault_t wp_pfn_shared(struct vm_fault *vmf)
   3279{
   3280	struct vm_area_struct *vma = vmf->vma;
   3281
   3282	if (vma->vm_ops && vma->vm_ops->pfn_mkwrite) {
   3283		vm_fault_t ret;
   3284
   3285		pte_unmap_unlock(vmf->pte, vmf->ptl);
   3286		vmf->flags |= FAULT_FLAG_MKWRITE;
   3287		ret = vma->vm_ops->pfn_mkwrite(vmf);
   3288		if (ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE))
   3289			return ret;
   3290		return finish_mkwrite_fault(vmf);
   3291	}
   3292	wp_page_reuse(vmf);
   3293	return VM_FAULT_WRITE;
   3294}
   3295
   3296static vm_fault_t wp_page_shared(struct vm_fault *vmf)
   3297	__releases(vmf->ptl)
   3298{
   3299	struct vm_area_struct *vma = vmf->vma;
   3300	vm_fault_t ret = VM_FAULT_WRITE;
   3301
   3302	get_page(vmf->page);
   3303
   3304	if (vma->vm_ops && vma->vm_ops->page_mkwrite) {
   3305		vm_fault_t tmp;
   3306
   3307		pte_unmap_unlock(vmf->pte, vmf->ptl);
   3308		tmp = do_page_mkwrite(vmf);
   3309		if (unlikely(!tmp || (tmp &
   3310				      (VM_FAULT_ERROR | VM_FAULT_NOPAGE)))) {
   3311			put_page(vmf->page);
   3312			return tmp;
   3313		}
   3314		tmp = finish_mkwrite_fault(vmf);
   3315		if (unlikely(tmp & (VM_FAULT_ERROR | VM_FAULT_NOPAGE))) {
   3316			unlock_page(vmf->page);
   3317			put_page(vmf->page);
   3318			return tmp;
   3319		}
   3320	} else {
   3321		wp_page_reuse(vmf);
   3322		lock_page(vmf->page);
   3323	}
   3324	ret |= fault_dirty_shared_page(vmf);
   3325	put_page(vmf->page);
   3326
   3327	return ret;
   3328}
   3329
   3330/*
   3331 * This routine handles present pages, when
   3332 * * users try to write to a shared page (FAULT_FLAG_WRITE)
   3333 * * GUP wants to take a R/O pin on a possibly shared anonymous page
   3334 *   (FAULT_FLAG_UNSHARE)
   3335 *
   3336 * It is done by copying the page to a new address and decrementing the
   3337 * shared-page counter for the old page.
   3338 *
   3339 * Note that this routine assumes that the protection checks have been
   3340 * done by the caller (the low-level page fault routine in most cases).
   3341 * Thus, with FAULT_FLAG_WRITE, we can safely just mark it writable once we've
   3342 * done any necessary COW.
   3343 *
   3344 * In case of FAULT_FLAG_WRITE, we also mark the page dirty at this point even
   3345 * though the page will change only once the write actually happens. This
   3346 * avoids a few races, and potentially makes it more efficient.
   3347 *
   3348 * We enter with non-exclusive mmap_lock (to exclude vma changes,
   3349 * but allow concurrent faults), with pte both mapped and locked.
   3350 * We return with mmap_lock still held, but pte unmapped and unlocked.
   3351 */
   3352static vm_fault_t do_wp_page(struct vm_fault *vmf)
   3353	__releases(vmf->ptl)
   3354{
   3355	const bool unshare = vmf->flags & FAULT_FLAG_UNSHARE;
   3356	struct vm_area_struct *vma = vmf->vma;
   3357
   3358	VM_BUG_ON(unshare && (vmf->flags & FAULT_FLAG_WRITE));
   3359	VM_BUG_ON(!unshare && !(vmf->flags & FAULT_FLAG_WRITE));
   3360
   3361	if (likely(!unshare)) {
   3362		if (userfaultfd_pte_wp(vma, *vmf->pte)) {
   3363			pte_unmap_unlock(vmf->pte, vmf->ptl);
   3364			return handle_userfault(vmf, VM_UFFD_WP);
   3365		}
   3366
   3367		/*
   3368		 * Userfaultfd write-protect can defer flushes. Ensure the TLB
   3369		 * is flushed in this case before copying.
   3370		 */
   3371		if (unlikely(userfaultfd_wp(vmf->vma) &&
   3372			     mm_tlb_flush_pending(vmf->vma->vm_mm)))
   3373			flush_tlb_page(vmf->vma, vmf->address);
   3374	}
   3375
   3376	vmf->page = vm_normal_page(vma, vmf->address, vmf->orig_pte);
   3377	if (!vmf->page) {
   3378		if (unlikely(unshare)) {
   3379			/* No anonymous page -> nothing to do. */
   3380			pte_unmap_unlock(vmf->pte, vmf->ptl);
   3381			return 0;
   3382		}
   3383
   3384		/*
   3385		 * VM_MIXEDMAP !pfn_valid() case, or VM_SOFTDIRTY clear on a
   3386		 * VM_PFNMAP VMA.
   3387		 *
   3388		 * We should not cow pages in a shared writeable mapping.
   3389		 * Just mark the pages writable and/or call ops->pfn_mkwrite.
   3390		 */
   3391		if ((vma->vm_flags & (VM_WRITE|VM_SHARED)) ==
   3392				     (VM_WRITE|VM_SHARED))
   3393			return wp_pfn_shared(vmf);
   3394
   3395		pte_unmap_unlock(vmf->pte, vmf->ptl);
   3396		return wp_page_copy(vmf);
   3397	}
   3398
   3399	/*
   3400	 * Take out anonymous pages first, anonymous shared vmas are
   3401	 * not dirty accountable.
   3402	 */
   3403	if (PageAnon(vmf->page)) {
   3404		struct page *page = vmf->page;
   3405
   3406		/*
   3407		 * If the page is exclusive to this process we must reuse the
   3408		 * page without further checks.
   3409		 */
   3410		if (PageAnonExclusive(page))
   3411			goto reuse;
   3412
   3413		/*
   3414		 * We have to verify under page lock: these early checks are
   3415		 * just an optimization to avoid locking the page and freeing
   3416		 * the swapcache if there is little hope that we can reuse.
   3417		 *
   3418		 * PageKsm() doesn't necessarily raise the page refcount.
   3419		 */
   3420		if (PageKsm(page) || page_count(page) > 3)
   3421			goto copy;
   3422		if (!PageLRU(page))
   3423			/*
   3424			 * Note: We cannot easily detect+handle references from
   3425			 * remote LRU pagevecs or references to PageLRU() pages.
   3426			 */
   3427			lru_add_drain();
   3428		if (page_count(page) > 1 + PageSwapCache(page))
   3429			goto copy;
   3430		if (!trylock_page(page))
   3431			goto copy;
   3432		if (PageSwapCache(page))
   3433			try_to_free_swap(page);
   3434		if (PageKsm(page) || page_count(page) != 1) {
   3435			unlock_page(page);
   3436			goto copy;
   3437		}
   3438		/*
   3439		 * Ok, we've got the only page reference from our mapping
   3440		 * and the page is locked, it's dark out, and we're wearing
   3441		 * sunglasses. Hit it.
   3442		 */
   3443		page_move_anon_rmap(page, vma);
   3444		unlock_page(page);
   3445reuse:
   3446		if (unlikely(unshare)) {
   3447			pte_unmap_unlock(vmf->pte, vmf->ptl);
   3448			return 0;
   3449		}
   3450		wp_page_reuse(vmf);
   3451		return VM_FAULT_WRITE;
   3452	} else if (unshare) {
   3453		/* No anonymous page -> nothing to do. */
   3454		pte_unmap_unlock(vmf->pte, vmf->ptl);
   3455		return 0;
   3456	} else if (unlikely((vma->vm_flags & (VM_WRITE|VM_SHARED)) ==
   3457					(VM_WRITE|VM_SHARED))) {
   3458		return wp_page_shared(vmf);
   3459	}
   3460copy:
   3461	/*
   3462	 * Ok, we need to copy. Oh, well..
   3463	 */
   3464	get_page(vmf->page);
   3465
   3466	pte_unmap_unlock(vmf->pte, vmf->ptl);
   3467#ifdef CONFIG_KSM
   3468	if (PageKsm(vmf->page))
   3469		count_vm_event(COW_KSM);
   3470#endif
   3471	return wp_page_copy(vmf);
   3472}
   3473
   3474static void unmap_mapping_range_vma(struct vm_area_struct *vma,
   3475		unsigned long start_addr, unsigned long end_addr,
   3476		struct zap_details *details)
   3477{
   3478	zap_page_range_single(vma, start_addr, end_addr - start_addr, details);
   3479}
   3480
   3481static inline void unmap_mapping_range_tree(struct rb_root_cached *root,
   3482					    pgoff_t first_index,
   3483					    pgoff_t last_index,
   3484					    struct zap_details *details)
   3485{
   3486	struct vm_area_struct *vma;
   3487	pgoff_t vba, vea, zba, zea;
   3488
   3489	vma_interval_tree_foreach(vma, root, first_index, last_index) {
   3490		vba = vma->vm_pgoff;
   3491		vea = vba + vma_pages(vma) - 1;
   3492		zba = max(first_index, vba);
   3493		zea = min(last_index, vea);
   3494
   3495		unmap_mapping_range_vma(vma,
   3496			((zba - vba) << PAGE_SHIFT) + vma->vm_start,
   3497			((zea - vba + 1) << PAGE_SHIFT) + vma->vm_start,
   3498				details);
   3499	}
   3500}
   3501
   3502/**
   3503 * unmap_mapping_folio() - Unmap single folio from processes.
   3504 * @folio: The locked folio to be unmapped.
   3505 *
   3506 * Unmap this folio from any userspace process which still has it mmaped.
   3507 * Typically, for efficiency, the range of nearby pages has already been
   3508 * unmapped by unmap_mapping_pages() or unmap_mapping_range().  But once
   3509 * truncation or invalidation holds the lock on a folio, it may find that
   3510 * the page has been remapped again: and then uses unmap_mapping_folio()
   3511 * to unmap it finally.
   3512 */
   3513void unmap_mapping_folio(struct folio *folio)
   3514{
   3515	struct address_space *mapping = folio->mapping;
   3516	struct zap_details details = { };
   3517	pgoff_t	first_index;
   3518	pgoff_t	last_index;
   3519
   3520	VM_BUG_ON(!folio_test_locked(folio));
   3521
   3522	first_index = folio->index;
   3523	last_index = folio->index + folio_nr_pages(folio) - 1;
   3524
   3525	details.even_cows = false;
   3526	details.single_folio = folio;
   3527	details.zap_flags = ZAP_FLAG_DROP_MARKER;
   3528
   3529	i_mmap_lock_read(mapping);
   3530	if (unlikely(!RB_EMPTY_ROOT(&mapping->i_mmap.rb_root)))
   3531		unmap_mapping_range_tree(&mapping->i_mmap, first_index,
   3532					 last_index, &details);
   3533	i_mmap_unlock_read(mapping);
   3534}
   3535
   3536/**
   3537 * unmap_mapping_pages() - Unmap pages from processes.
   3538 * @mapping: The address space containing pages to be unmapped.
   3539 * @start: Index of first page to be unmapped.
   3540 * @nr: Number of pages to be unmapped.  0 to unmap to end of file.
   3541 * @even_cows: Whether to unmap even private COWed pages.
   3542 *
   3543 * Unmap the pages in this address space from any userspace process which
   3544 * has them mmaped.  Generally, you want to remove COWed pages as well when
   3545 * a file is being truncated, but not when invalidating pages from the page
   3546 * cache.
   3547 */
   3548void unmap_mapping_pages(struct address_space *mapping, pgoff_t start,
   3549		pgoff_t nr, bool even_cows)
   3550{
   3551	struct zap_details details = { };
   3552	pgoff_t	first_index = start;
   3553	pgoff_t	last_index = start + nr - 1;
   3554
   3555	details.even_cows = even_cows;
   3556	if (last_index < first_index)
   3557		last_index = ULONG_MAX;
   3558
   3559	i_mmap_lock_read(mapping);
   3560	if (unlikely(!RB_EMPTY_ROOT(&mapping->i_mmap.rb_root)))
   3561		unmap_mapping_range_tree(&mapping->i_mmap, first_index,
   3562					 last_index, &details);
   3563	i_mmap_unlock_read(mapping);
   3564}
   3565EXPORT_SYMBOL_GPL(unmap_mapping_pages);
   3566
   3567/**
   3568 * unmap_mapping_range - unmap the portion of all mmaps in the specified
   3569 * address_space corresponding to the specified byte range in the underlying
   3570 * file.
   3571 *
   3572 * @mapping: the address space containing mmaps to be unmapped.
   3573 * @holebegin: byte in first page to unmap, relative to the start of
   3574 * the underlying file.  This will be rounded down to a PAGE_SIZE
   3575 * boundary.  Note that this is different from truncate_pagecache(), which
   3576 * must keep the partial page.  In contrast, we must get rid of
   3577 * partial pages.
   3578 * @holelen: size of prospective hole in bytes.  This will be rounded
   3579 * up to a PAGE_SIZE boundary.  A holelen of zero truncates to the
   3580 * end of the file.
   3581 * @even_cows: 1 when truncating a file, unmap even private COWed pages;
   3582 * but 0 when invalidating pagecache, don't throw away private data.
   3583 */
   3584void unmap_mapping_range(struct address_space *mapping,
   3585		loff_t const holebegin, loff_t const holelen, int even_cows)
   3586{
   3587	pgoff_t hba = holebegin >> PAGE_SHIFT;
   3588	pgoff_t hlen = (holelen + PAGE_SIZE - 1) >> PAGE_SHIFT;
   3589
   3590	/* Check for overflow. */
   3591	if (sizeof(holelen) > sizeof(hlen)) {
   3592		long long holeend =
   3593			(holebegin + holelen + PAGE_SIZE - 1) >> PAGE_SHIFT;
   3594		if (holeend & ~(long long)ULONG_MAX)
   3595			hlen = ULONG_MAX - hba + 1;
   3596	}
   3597
   3598	unmap_mapping_pages(mapping, hba, hlen, even_cows);
   3599}
   3600EXPORT_SYMBOL(unmap_mapping_range);
   3601
   3602/*
   3603 * Restore a potential device exclusive pte to a working pte entry
   3604 */
   3605static vm_fault_t remove_device_exclusive_entry(struct vm_fault *vmf)
   3606{
   3607	struct page *page = vmf->page;
   3608	struct vm_area_struct *vma = vmf->vma;
   3609	struct mmu_notifier_range range;
   3610
   3611	if (!lock_page_or_retry(page, vma->vm_mm, vmf->flags))
   3612		return VM_FAULT_RETRY;
   3613	mmu_notifier_range_init_owner(&range, MMU_NOTIFY_EXCLUSIVE, 0, vma,
   3614				vma->vm_mm, vmf->address & PAGE_MASK,
   3615				(vmf->address & PAGE_MASK) + PAGE_SIZE, NULL);
   3616	mmu_notifier_invalidate_range_start(&range);
   3617
   3618	vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address,
   3619				&vmf->ptl);
   3620	if (likely(pte_same(*vmf->pte, vmf->orig_pte)))
   3621		restore_exclusive_pte(vma, page, vmf->address, vmf->pte);
   3622
   3623	pte_unmap_unlock(vmf->pte, vmf->ptl);
   3624	unlock_page(page);
   3625
   3626	mmu_notifier_invalidate_range_end(&range);
   3627	return 0;
   3628}
   3629
   3630static inline bool should_try_to_free_swap(struct page *page,
   3631					   struct vm_area_struct *vma,
   3632					   unsigned int fault_flags)
   3633{
   3634	if (!PageSwapCache(page))
   3635		return false;
   3636	if (mem_cgroup_swap_full(page) || (vma->vm_flags & VM_LOCKED) ||
   3637	    PageMlocked(page))
   3638		return true;
   3639	/*
   3640	 * If we want to map a page that's in the swapcache writable, we
   3641	 * have to detect via the refcount if we're really the exclusive
   3642	 * user. Try freeing the swapcache to get rid of the swapcache
   3643	 * reference only in case it's likely that we'll be the exlusive user.
   3644	 */
   3645	return (fault_flags & FAULT_FLAG_WRITE) && !PageKsm(page) &&
   3646		page_count(page) == 2;
   3647}
   3648
   3649static vm_fault_t pte_marker_clear(struct vm_fault *vmf)
   3650{
   3651	vmf->pte = pte_offset_map_lock(vmf->vma->vm_mm, vmf->pmd,
   3652				       vmf->address, &vmf->ptl);
   3653	/*
   3654	 * Be careful so that we will only recover a special uffd-wp pte into a
   3655	 * none pte.  Otherwise it means the pte could have changed, so retry.
   3656	 */
   3657	if (is_pte_marker(*vmf->pte))
   3658		pte_clear(vmf->vma->vm_mm, vmf->address, vmf->pte);
   3659	pte_unmap_unlock(vmf->pte, vmf->ptl);
   3660	return 0;
   3661}
   3662
   3663/*
   3664 * This is actually a page-missing access, but with uffd-wp special pte
   3665 * installed.  It means this pte was wr-protected before being unmapped.
   3666 */
   3667static vm_fault_t pte_marker_handle_uffd_wp(struct vm_fault *vmf)
   3668{
   3669	/*
   3670	 * Just in case there're leftover special ptes even after the region
   3671	 * got unregistered - we can simply clear them.  We can also do that
   3672	 * proactively when e.g. when we do UFFDIO_UNREGISTER upon some uffd-wp
   3673	 * ranges, but it should be more efficient to be done lazily here.
   3674	 */
   3675	if (unlikely(!userfaultfd_wp(vmf->vma) || vma_is_anonymous(vmf->vma)))
   3676		return pte_marker_clear(vmf);
   3677
   3678	/* do_fault() can handle pte markers too like none pte */
   3679	return do_fault(vmf);
   3680}
   3681
   3682static vm_fault_t handle_pte_marker(struct vm_fault *vmf)
   3683{
   3684	swp_entry_t entry = pte_to_swp_entry(vmf->orig_pte);
   3685	unsigned long marker = pte_marker_get(entry);
   3686
   3687	/*
   3688	 * PTE markers should always be with file-backed memories, and the
   3689	 * marker should never be empty.  If anything weird happened, the best
   3690	 * thing to do is to kill the process along with its mm.
   3691	 */
   3692	if (WARN_ON_ONCE(vma_is_anonymous(vmf->vma) || !marker))
   3693		return VM_FAULT_SIGBUS;
   3694
   3695	if (pte_marker_entry_uffd_wp(entry))
   3696		return pte_marker_handle_uffd_wp(vmf);
   3697
   3698	/* This is an unknown pte marker */
   3699	return VM_FAULT_SIGBUS;
   3700}
   3701
   3702/*
   3703 * We enter with non-exclusive mmap_lock (to exclude vma changes,
   3704 * but allow concurrent faults), and pte mapped but not yet locked.
   3705 * We return with pte unmapped and unlocked.
   3706 *
   3707 * We return with the mmap_lock locked or unlocked in the same cases
   3708 * as does filemap_fault().
   3709 */
   3710vm_fault_t do_swap_page(struct vm_fault *vmf)
   3711{
   3712	struct vm_area_struct *vma = vmf->vma;
   3713	struct page *page = NULL, *swapcache;
   3714	struct swap_info_struct *si = NULL;
   3715	rmap_t rmap_flags = RMAP_NONE;
   3716	bool exclusive = false;
   3717	swp_entry_t entry;
   3718	pte_t pte;
   3719	int locked;
   3720	vm_fault_t ret = 0;
   3721	void *shadow = NULL;
   3722
   3723	if (!pte_unmap_same(vmf))
   3724		goto out;
   3725
   3726	entry = pte_to_swp_entry(vmf->orig_pte);
   3727	if (unlikely(non_swap_entry(entry))) {
   3728		if (is_migration_entry(entry)) {
   3729			migration_entry_wait(vma->vm_mm, vmf->pmd,
   3730					     vmf->address);
   3731		} else if (is_device_exclusive_entry(entry)) {
   3732			vmf->page = pfn_swap_entry_to_page(entry);
   3733			ret = remove_device_exclusive_entry(vmf);
   3734		} else if (is_device_private_entry(entry)) {
   3735			vmf->page = pfn_swap_entry_to_page(entry);
   3736			ret = vmf->page->pgmap->ops->migrate_to_ram(vmf);
   3737		} else if (is_hwpoison_entry(entry)) {
   3738			ret = VM_FAULT_HWPOISON;
   3739		} else if (is_swapin_error_entry(entry)) {
   3740			ret = VM_FAULT_SIGBUS;
   3741		} else if (is_pte_marker_entry(entry)) {
   3742			ret = handle_pte_marker(vmf);
   3743		} else {
   3744			print_bad_pte(vma, vmf->address, vmf->orig_pte, NULL);
   3745			ret = VM_FAULT_SIGBUS;
   3746		}
   3747		goto out;
   3748	}
   3749
   3750	/* Prevent swapoff from happening to us. */
   3751	si = get_swap_device(entry);
   3752	if (unlikely(!si))
   3753		goto out;
   3754
   3755	page = lookup_swap_cache(entry, vma, vmf->address);
   3756	swapcache = page;
   3757
   3758	if (!page) {
   3759		if (data_race(si->flags & SWP_SYNCHRONOUS_IO) &&
   3760		    __swap_count(entry) == 1) {
   3761			/* skip swapcache */
   3762			page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma,
   3763							vmf->address);
   3764			if (page) {
   3765				__SetPageLocked(page);
   3766				__SetPageSwapBacked(page);
   3767
   3768				if (mem_cgroup_swapin_charge_page(page,
   3769					vma->vm_mm, GFP_KERNEL, entry)) {
   3770					ret = VM_FAULT_OOM;
   3771					goto out_page;
   3772				}
   3773				mem_cgroup_swapin_uncharge_swap(entry);
   3774
   3775				shadow = get_shadow_from_swap_cache(entry);
   3776				if (shadow)
   3777					workingset_refault(page_folio(page),
   3778								shadow);
   3779
   3780				lru_cache_add(page);
   3781
   3782				/* To provide entry to swap_readpage() */
   3783				set_page_private(page, entry.val);
   3784				swap_readpage(page, true, NULL);
   3785				set_page_private(page, 0);
   3786			}
   3787		} else {
   3788			page = swapin_readahead(entry, GFP_HIGHUSER_MOVABLE,
   3789						vmf);
   3790			swapcache = page;
   3791		}
   3792
   3793		if (!page) {
   3794			/*
   3795			 * Back out if somebody else faulted in this pte
   3796			 * while we released the pte lock.
   3797			 */
   3798			vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
   3799					vmf->address, &vmf->ptl);
   3800			if (likely(pte_same(*vmf->pte, vmf->orig_pte)))
   3801				ret = VM_FAULT_OOM;
   3802			goto unlock;
   3803		}
   3804
   3805		/* Had to read the page from swap area: Major fault */
   3806		ret = VM_FAULT_MAJOR;
   3807		count_vm_event(PGMAJFAULT);
   3808		count_memcg_event_mm(vma->vm_mm, PGMAJFAULT);
   3809	} else if (PageHWPoison(page)) {
   3810		/*
   3811		 * hwpoisoned dirty swapcache pages are kept for killing
   3812		 * owner processes (which may be unknown at hwpoison time)
   3813		 */
   3814		ret = VM_FAULT_HWPOISON;
   3815		goto out_release;
   3816	}
   3817
   3818	locked = lock_page_or_retry(page, vma->vm_mm, vmf->flags);
   3819
   3820	if (!locked) {
   3821		ret |= VM_FAULT_RETRY;
   3822		goto out_release;
   3823	}
   3824
   3825	if (swapcache) {
   3826		/*
   3827		 * Make sure try_to_free_swap or swapoff did not release the
   3828		 * swapcache from under us.  The page pin, and pte_same test
   3829		 * below, are not enough to exclude that.  Even if it is still
   3830		 * swapcache, we need to check that the page's swap has not
   3831		 * changed.
   3832		 */
   3833		if (unlikely(!PageSwapCache(page) ||
   3834			     page_private(page) != entry.val))
   3835			goto out_page;
   3836
   3837		/*
   3838		 * KSM sometimes has to copy on read faults, for example, if
   3839		 * page->index of !PageKSM() pages would be nonlinear inside the
   3840		 * anon VMA -- PageKSM() is lost on actual swapout.
   3841		 */
   3842		page = ksm_might_need_to_copy(page, vma, vmf->address);
   3843		if (unlikely(!page)) {
   3844			ret = VM_FAULT_OOM;
   3845			page = swapcache;
   3846			goto out_page;
   3847		}
   3848
   3849		/*
   3850		 * If we want to map a page that's in the swapcache writable, we
   3851		 * have to detect via the refcount if we're really the exclusive
   3852		 * owner. Try removing the extra reference from the local LRU
   3853		 * pagevecs if required.
   3854		 */
   3855		if ((vmf->flags & FAULT_FLAG_WRITE) && page == swapcache &&
   3856		    !PageKsm(page) && !PageLRU(page))
   3857			lru_add_drain();
   3858	}
   3859
   3860	cgroup_throttle_swaprate(page, GFP_KERNEL);
   3861
   3862	/*
   3863	 * Back out if somebody else already faulted in this pte.
   3864	 */
   3865	vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address,
   3866			&vmf->ptl);
   3867	if (unlikely(!pte_same(*vmf->pte, vmf->orig_pte)))
   3868		goto out_nomap;
   3869
   3870	if (unlikely(!PageUptodate(page))) {
   3871		ret = VM_FAULT_SIGBUS;
   3872		goto out_nomap;
   3873	}
   3874
   3875	/*
   3876	 * PG_anon_exclusive reuses PG_mappedtodisk for anon pages. A swap pte
   3877	 * must never point at an anonymous page in the swapcache that is
   3878	 * PG_anon_exclusive. Sanity check that this holds and especially, that
   3879	 * no filesystem set PG_mappedtodisk on a page in the swapcache. Sanity
   3880	 * check after taking the PT lock and making sure that nobody
   3881	 * concurrently faulted in this page and set PG_anon_exclusive.
   3882	 */
   3883	BUG_ON(!PageAnon(page) && PageMappedToDisk(page));
   3884	BUG_ON(PageAnon(page) && PageAnonExclusive(page));
   3885
   3886	/*
   3887	 * Check under PT lock (to protect against concurrent fork() sharing
   3888	 * the swap entry concurrently) for certainly exclusive pages.
   3889	 */
   3890	if (!PageKsm(page)) {
   3891		/*
   3892		 * Note that pte_swp_exclusive() == false for architectures
   3893		 * without __HAVE_ARCH_PTE_SWP_EXCLUSIVE.
   3894		 */
   3895		exclusive = pte_swp_exclusive(vmf->orig_pte);
   3896		if (page != swapcache) {
   3897			/*
   3898			 * We have a fresh page that is not exposed to the
   3899			 * swapcache -> certainly exclusive.
   3900			 */
   3901			exclusive = true;
   3902		} else if (exclusive && PageWriteback(page) &&
   3903			  data_race(si->flags & SWP_STABLE_WRITES)) {
   3904			/*
   3905			 * This is tricky: not all swap backends support
   3906			 * concurrent page modifications while under writeback.
   3907			 *
   3908			 * So if we stumble over such a page in the swapcache
   3909			 * we must not set the page exclusive, otherwise we can
   3910			 * map it writable without further checks and modify it
   3911			 * while still under writeback.
   3912			 *
   3913			 * For these problematic swap backends, simply drop the
   3914			 * exclusive marker: this is perfectly fine as we start
   3915			 * writeback only if we fully unmapped the page and
   3916			 * there are no unexpected references on the page after
   3917			 * unmapping succeeded. After fully unmapped, no
   3918			 * further GUP references (FOLL_GET and FOLL_PIN) can
   3919			 * appear, so dropping the exclusive marker and mapping
   3920			 * it only R/O is fine.
   3921			 */
   3922			exclusive = false;
   3923		}
   3924	}
   3925
   3926	/*
   3927	 * Remove the swap entry and conditionally try to free up the swapcache.
   3928	 * We're already holding a reference on the page but haven't mapped it
   3929	 * yet.
   3930	 */
   3931	swap_free(entry);
   3932	if (should_try_to_free_swap(page, vma, vmf->flags))
   3933		try_to_free_swap(page);
   3934
   3935	inc_mm_counter_fast(vma->vm_mm, MM_ANONPAGES);
   3936	dec_mm_counter_fast(vma->vm_mm, MM_SWAPENTS);
   3937	pte = mk_pte(page, vma->vm_page_prot);
   3938
   3939	/*
   3940	 * Same logic as in do_wp_page(); however, optimize for pages that are
   3941	 * certainly not shared either because we just allocated them without
   3942	 * exposing them to the swapcache or because the swap entry indicates
   3943	 * exclusivity.
   3944	 */
   3945	if (!PageKsm(page) && (exclusive || page_count(page) == 1)) {
   3946		if (vmf->flags & FAULT_FLAG_WRITE) {
   3947			pte = maybe_mkwrite(pte_mkdirty(pte), vma);
   3948			vmf->flags &= ~FAULT_FLAG_WRITE;
   3949			ret |= VM_FAULT_WRITE;
   3950		}
   3951		rmap_flags |= RMAP_EXCLUSIVE;
   3952	}
   3953	flush_icache_page(vma, page);
   3954	if (pte_swp_soft_dirty(vmf->orig_pte))
   3955		pte = pte_mksoft_dirty(pte);
   3956	if (pte_swp_uffd_wp(vmf->orig_pte)) {
   3957		pte = pte_mkuffd_wp(pte);
   3958		pte = pte_wrprotect(pte);
   3959	}
   3960	vmf->orig_pte = pte;
   3961
   3962	/* ksm created a completely new copy */
   3963	if (unlikely(page != swapcache && swapcache)) {
   3964		page_add_new_anon_rmap(page, vma, vmf->address);
   3965		lru_cache_add_inactive_or_unevictable(page, vma);
   3966	} else {
   3967		page_add_anon_rmap(page, vma, vmf->address, rmap_flags);
   3968	}
   3969
   3970	VM_BUG_ON(!PageAnon(page) || (pte_write(pte) && !PageAnonExclusive(page)));
   3971	set_pte_at(vma->vm_mm, vmf->address, vmf->pte, pte);
   3972	arch_do_swap_page(vma->vm_mm, vma, vmf->address, pte, vmf->orig_pte);
   3973
   3974	unlock_page(page);
   3975	if (page != swapcache && swapcache) {
   3976		/*
   3977		 * Hold the lock to avoid the swap entry to be reused
   3978		 * until we take the PT lock for the pte_same() check
   3979		 * (to avoid false positives from pte_same). For
   3980		 * further safety release the lock after the swap_free
   3981		 * so that the swap count won't change under a
   3982		 * parallel locked swapcache.
   3983		 */
   3984		unlock_page(swapcache);
   3985		put_page(swapcache);
   3986	}
   3987
   3988	if (vmf->flags & FAULT_FLAG_WRITE) {
   3989		ret |= do_wp_page(vmf);
   3990		if (ret & VM_FAULT_ERROR)
   3991			ret &= VM_FAULT_ERROR;
   3992		goto out;
   3993	}
   3994
   3995	/* No need to invalidate - it was non-present before */
   3996	update_mmu_cache(vma, vmf->address, vmf->pte);
   3997unlock:
   3998	pte_unmap_unlock(vmf->pte, vmf->ptl);
   3999out:
   4000	if (si)
   4001		put_swap_device(si);
   4002	return ret;
   4003out_nomap:
   4004	pte_unmap_unlock(vmf->pte, vmf->ptl);
   4005out_page:
   4006	unlock_page(page);
   4007out_release:
   4008	put_page(page);
   4009	if (page != swapcache && swapcache) {
   4010		unlock_page(swapcache);
   4011		put_page(swapcache);
   4012	}
   4013	if (si)
   4014		put_swap_device(si);
   4015	return ret;
   4016}
   4017
   4018/*
   4019 * We enter with non-exclusive mmap_lock (to exclude vma changes,
   4020 * but allow concurrent faults), and pte mapped but not yet locked.
   4021 * We return with mmap_lock still held, but pte unmapped and unlocked.
   4022 */
   4023static vm_fault_t do_anonymous_page(struct vm_fault *vmf)
   4024{
   4025	struct vm_area_struct *vma = vmf->vma;
   4026	struct page *page;
   4027	vm_fault_t ret = 0;
   4028	pte_t entry;
   4029
   4030	/* File mapping without ->vm_ops ? */
   4031	if (vma->vm_flags & VM_SHARED)
   4032		return VM_FAULT_SIGBUS;
   4033
   4034	/*
   4035	 * Use pte_alloc() instead of pte_alloc_map().  We can't run
   4036	 * pte_offset_map() on pmds where a huge pmd might be created
   4037	 * from a different thread.
   4038	 *
   4039	 * pte_alloc_map() is safe to use under mmap_write_lock(mm) or when
   4040	 * parallel threads are excluded by other means.
   4041	 *
   4042	 * Here we only have mmap_read_lock(mm).
   4043	 */
   4044	if (pte_alloc(vma->vm_mm, vmf->pmd))
   4045		return VM_FAULT_OOM;
   4046
   4047	/* See comment in handle_pte_fault() */
   4048	if (unlikely(pmd_trans_unstable(vmf->pmd)))
   4049		return 0;
   4050
   4051	/* Use the zero-page for reads */
   4052	if (!(vmf->flags & FAULT_FLAG_WRITE) &&
   4053			!mm_forbids_zeropage(vma->vm_mm)) {
   4054		entry = pte_mkspecial(pfn_pte(my_zero_pfn(vmf->address),
   4055						vma->vm_page_prot));
   4056		vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
   4057				vmf->address, &vmf->ptl);
   4058		if (!pte_none(*vmf->pte)) {
   4059			update_mmu_tlb(vma, vmf->address, vmf->pte);
   4060			goto unlock;
   4061		}
   4062		ret = check_stable_address_space(vma->vm_mm);
   4063		if (ret)
   4064			goto unlock;
   4065		/* Deliver the page fault to userland, check inside PT lock */
   4066		if (userfaultfd_missing(vma)) {
   4067			pte_unmap_unlock(vmf->pte, vmf->ptl);
   4068			return handle_userfault(vmf, VM_UFFD_MISSING);
   4069		}
   4070		goto setpte;
   4071	}
   4072
   4073	/* Allocate our own private page. */
   4074	if (unlikely(anon_vma_prepare(vma)))
   4075		goto oom;
   4076	page = alloc_zeroed_user_highpage_movable(vma, vmf->address);
   4077	if (!page)
   4078		goto oom;
   4079
   4080	if (mem_cgroup_charge(page_folio(page), vma->vm_mm, GFP_KERNEL))
   4081		goto oom_free_page;
   4082	cgroup_throttle_swaprate(page, GFP_KERNEL);
   4083
   4084	/*
   4085	 * The memory barrier inside __SetPageUptodate makes sure that
   4086	 * preceding stores to the page contents become visible before
   4087	 * the set_pte_at() write.
   4088	 */
   4089	__SetPageUptodate(page);
   4090
   4091	entry = mk_pte(page, vma->vm_page_prot);
   4092	entry = pte_sw_mkyoung(entry);
   4093	if (vma->vm_flags & VM_WRITE)
   4094		entry = pte_mkwrite(pte_mkdirty(entry));
   4095
   4096	vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address,
   4097			&vmf->ptl);
   4098	if (!pte_none(*vmf->pte)) {
   4099		update_mmu_cache(vma, vmf->address, vmf->pte);
   4100		goto release;
   4101	}
   4102
   4103	ret = check_stable_address_space(vma->vm_mm);
   4104	if (ret)
   4105		goto release;
   4106
   4107	/* Deliver the page fault to userland, check inside PT lock */
   4108	if (userfaultfd_missing(vma)) {
   4109		pte_unmap_unlock(vmf->pte, vmf->ptl);
   4110		put_page(page);
   4111		return handle_userfault(vmf, VM_UFFD_MISSING);
   4112	}
   4113
   4114	inc_mm_counter_fast(vma->vm_mm, MM_ANONPAGES);
   4115	page_add_new_anon_rmap(page, vma, vmf->address);
   4116	lru_cache_add_inactive_or_unevictable(page, vma);
   4117setpte:
   4118	set_pte_at(vma->vm_mm, vmf->address, vmf->pte, entry);
   4119
   4120	/* No need to invalidate - it was non-present before */
   4121	update_mmu_cache(vma, vmf->address, vmf->pte);
   4122unlock:
   4123	pte_unmap_unlock(vmf->pte, vmf->ptl);
   4124	return ret;
   4125release:
   4126	put_page(page);
   4127	goto unlock;
   4128oom_free_page:
   4129	put_page(page);
   4130oom:
   4131	return VM_FAULT_OOM;
   4132}
   4133
   4134/*
   4135 * The mmap_lock must have been held on entry, and may have been
   4136 * released depending on flags and vma->vm_ops->fault() return value.
   4137 * See filemap_fault() and __lock_page_retry().
   4138 */
   4139static vm_fault_t __do_fault(struct vm_fault *vmf)
   4140{
   4141	struct vm_area_struct *vma = vmf->vma;
   4142	vm_fault_t ret;
   4143
   4144	/*
   4145	 * Preallocate pte before we take page_lock because this might lead to
   4146	 * deadlocks for memcg reclaim which waits for pages under writeback:
   4147	 *				lock_page(A)
   4148	 *				SetPageWriteback(A)
   4149	 *				unlock_page(A)
   4150	 * lock_page(B)
   4151	 *				lock_page(B)
   4152	 * pte_alloc_one
   4153	 *   shrink_page_list
   4154	 *     wait_on_page_writeback(A)
   4155	 *				SetPageWriteback(B)
   4156	 *				unlock_page(B)
   4157	 *				# flush A, B to clear the writeback
   4158	 */
   4159	if (pmd_none(*vmf->pmd) && !vmf->prealloc_pte) {
   4160		vmf->prealloc_pte = pte_alloc_one(vma->vm_mm);
   4161		if (!vmf->prealloc_pte)
   4162			return VM_FAULT_OOM;
   4163	}
   4164
   4165	ret = vma->vm_ops->fault(vmf);
   4166	if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY |
   4167			    VM_FAULT_DONE_COW)))
   4168		return ret;
   4169
   4170	if (unlikely(PageHWPoison(vmf->page))) {
   4171		struct page *page = vmf->page;
   4172		vm_fault_t poisonret = VM_FAULT_HWPOISON;
   4173		if (ret & VM_FAULT_LOCKED) {
   4174			if (page_mapped(page))
   4175				unmap_mapping_pages(page_mapping(page),
   4176						    page->index, 1, false);
   4177			/* Retry if a clean page was removed from the cache. */
   4178			if (invalidate_inode_page(page))
   4179				poisonret = VM_FAULT_NOPAGE;
   4180			unlock_page(page);
   4181		}
   4182		put_page(page);
   4183		vmf->page = NULL;
   4184		return poisonret;
   4185	}
   4186
   4187	if (unlikely(!(ret & VM_FAULT_LOCKED)))
   4188		lock_page(vmf->page);
   4189	else
   4190		VM_BUG_ON_PAGE(!PageLocked(vmf->page), vmf->page);
   4191
   4192	return ret;
   4193}
   4194
   4195#ifdef CONFIG_TRANSPARENT_HUGEPAGE
   4196static void deposit_prealloc_pte(struct vm_fault *vmf)
   4197{
   4198	struct vm_area_struct *vma = vmf->vma;
   4199
   4200	pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, vmf->prealloc_pte);
   4201	/*
   4202	 * We are going to consume the prealloc table,
   4203	 * count that as nr_ptes.
   4204	 */
   4205	mm_inc_nr_ptes(vma->vm_mm);
   4206	vmf->prealloc_pte = NULL;
   4207}
   4208
   4209vm_fault_t do_set_pmd(struct vm_fault *vmf, struct page *page)
   4210{
   4211	struct vm_area_struct *vma = vmf->vma;
   4212	bool write = vmf->flags & FAULT_FLAG_WRITE;
   4213	unsigned long haddr = vmf->address & HPAGE_PMD_MASK;
   4214	pmd_t entry;
   4215	int i;
   4216	vm_fault_t ret = VM_FAULT_FALLBACK;
   4217
   4218	if (!transhuge_vma_suitable(vma, haddr))
   4219		return ret;
   4220
   4221	page = compound_head(page);
   4222	if (compound_order(page) != HPAGE_PMD_ORDER)
   4223		return ret;
   4224
   4225	/*
   4226	 * Just backoff if any subpage of a THP is corrupted otherwise
   4227	 * the corrupted page may mapped by PMD silently to escape the
   4228	 * check.  This kind of THP just can be PTE mapped.  Access to
   4229	 * the corrupted subpage should trigger SIGBUS as expected.
   4230	 */
   4231	if (unlikely(PageHasHWPoisoned(page)))
   4232		return ret;
   4233
   4234	/*
   4235	 * Archs like ppc64 need additional space to store information
   4236	 * related to pte entry. Use the preallocated table for that.
   4237	 */
   4238	if (arch_needs_pgtable_deposit() && !vmf->prealloc_pte) {
   4239		vmf->prealloc_pte = pte_alloc_one(vma->vm_mm);
   4240		if (!vmf->prealloc_pte)
   4241			return VM_FAULT_OOM;
   4242	}
   4243
   4244	vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);
   4245	if (unlikely(!pmd_none(*vmf->pmd)))
   4246		goto out;
   4247
   4248	for (i = 0; i < HPAGE_PMD_NR; i++)
   4249		flush_icache_page(vma, page + i);
   4250
   4251	entry = mk_huge_pmd(page, vma->vm_page_prot);
   4252	if (write)
   4253		entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
   4254
   4255	add_mm_counter(vma->vm_mm, mm_counter_file(page), HPAGE_PMD_NR);
   4256	page_add_file_rmap(page, vma, true);
   4257
   4258	/*
   4259	 * deposit and withdraw with pmd lock held
   4260	 */
   4261	if (arch_needs_pgtable_deposit())
   4262		deposit_prealloc_pte(vmf);
   4263
   4264	set_pmd_at(vma->vm_mm, haddr, vmf->pmd, entry);
   4265
   4266	update_mmu_cache_pmd(vma, haddr, vmf->pmd);
   4267
   4268	/* fault is handled */
   4269	ret = 0;
   4270	count_vm_event(THP_FILE_MAPPED);
   4271out:
   4272	spin_unlock(vmf->ptl);
   4273	return ret;
   4274}
   4275#else
   4276vm_fault_t do_set_pmd(struct vm_fault *vmf, struct page *page)
   4277{
   4278	return VM_FAULT_FALLBACK;
   4279}
   4280#endif
   4281
   4282void do_set_pte(struct vm_fault *vmf, struct page *page, unsigned long addr)
   4283{
   4284	struct vm_area_struct *vma = vmf->vma;
   4285	bool uffd_wp = pte_marker_uffd_wp(vmf->orig_pte);
   4286	bool write = vmf->flags & FAULT_FLAG_WRITE;
   4287	bool prefault = vmf->address != addr;
   4288	pte_t entry;
   4289
   4290	flush_icache_page(vma, page);
   4291	entry = mk_pte(page, vma->vm_page_prot);
   4292
   4293	if (prefault && arch_wants_old_prefaulted_pte())
   4294		entry = pte_mkold(entry);
   4295	else
   4296		entry = pte_sw_mkyoung(entry);
   4297
   4298	if (write)
   4299		entry = maybe_mkwrite(pte_mkdirty(entry), vma);
   4300	if (unlikely(uffd_wp))
   4301		entry = pte_mkuffd_wp(pte_wrprotect(entry));
   4302	/* copy-on-write page */
   4303	if (write && !(vma->vm_flags & VM_SHARED)) {
   4304		inc_mm_counter_fast(vma->vm_mm, MM_ANONPAGES);
   4305		page_add_new_anon_rmap(page, vma, addr);
   4306		lru_cache_add_inactive_or_unevictable(page, vma);
   4307	} else {
   4308		inc_mm_counter_fast(vma->vm_mm, mm_counter_file(page));
   4309		page_add_file_rmap(page, vma, false);
   4310	}
   4311	set_pte_at(vma->vm_mm, addr, vmf->pte, entry);
   4312}
   4313
   4314static bool vmf_pte_changed(struct vm_fault *vmf)
   4315{
   4316	if (vmf->flags & FAULT_FLAG_ORIG_PTE_VALID)
   4317		return !pte_same(*vmf->pte, vmf->orig_pte);
   4318
   4319	return !pte_none(*vmf->pte);
   4320}
   4321
   4322/**
   4323 * finish_fault - finish page fault once we have prepared the page to fault
   4324 *
   4325 * @vmf: structure describing the fault
   4326 *
   4327 * This function handles all that is needed to finish a page fault once the
   4328 * page to fault in is prepared. It handles locking of PTEs, inserts PTE for
   4329 * given page, adds reverse page mapping, handles memcg charges and LRU
   4330 * addition.
   4331 *
   4332 * The function expects the page to be locked and on success it consumes a
   4333 * reference of a page being mapped (for the PTE which maps it).
   4334 *
   4335 * Return: %0 on success, %VM_FAULT_ code in case of error.
   4336 */
   4337vm_fault_t finish_fault(struct vm_fault *vmf)
   4338{
   4339	struct vm_area_struct *vma = vmf->vma;
   4340	struct page *page;
   4341	vm_fault_t ret;
   4342
   4343	/* Did we COW the page? */
   4344	if ((vmf->flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED))
   4345		page = vmf->cow_page;
   4346	else
   4347		page = vmf->page;
   4348
   4349	/*
   4350	 * check even for read faults because we might have lost our CoWed
   4351	 * page
   4352	 */
   4353	if (!(vma->vm_flags & VM_SHARED)) {
   4354		ret = check_stable_address_space(vma->vm_mm);
   4355		if (ret)
   4356			return ret;
   4357	}
   4358
   4359	if (pmd_none(*vmf->pmd)) {
   4360		if (PageTransCompound(page)) {
   4361			ret = do_set_pmd(vmf, page);
   4362			if (ret != VM_FAULT_FALLBACK)
   4363				return ret;
   4364		}
   4365
   4366		if (vmf->prealloc_pte)
   4367			pmd_install(vma->vm_mm, vmf->pmd, &vmf->prealloc_pte);
   4368		else if (unlikely(pte_alloc(vma->vm_mm, vmf->pmd)))
   4369			return VM_FAULT_OOM;
   4370	}
   4371
   4372	/* See comment in handle_pte_fault() */
   4373	if (pmd_devmap_trans_unstable(vmf->pmd))
   4374		return 0;
   4375
   4376	vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
   4377				      vmf->address, &vmf->ptl);
   4378	ret = 0;
   4379	/* Re-check under ptl */
   4380	if (likely(!vmf_pte_changed(vmf)))
   4381		do_set_pte(vmf, page, vmf->address);
   4382	else
   4383		ret = VM_FAULT_NOPAGE;
   4384
   4385	update_mmu_tlb(vma, vmf->address, vmf->pte);
   4386	pte_unmap_unlock(vmf->pte, vmf->ptl);
   4387	return ret;
   4388}
   4389
   4390static unsigned long fault_around_bytes __read_mostly =
   4391	rounddown_pow_of_two(65536);
   4392
   4393#ifdef CONFIG_DEBUG_FS
   4394static int fault_around_bytes_get(void *data, u64 *val)
   4395{
   4396	*val = fault_around_bytes;
   4397	return 0;
   4398}
   4399
   4400/*
   4401 * fault_around_bytes must be rounded down to the nearest page order as it's
   4402 * what do_fault_around() expects to see.
   4403 */
   4404static int fault_around_bytes_set(void *data, u64 val)
   4405{
   4406	if (val / PAGE_SIZE > PTRS_PER_PTE)
   4407		return -EINVAL;
   4408	if (val > PAGE_SIZE)
   4409		fault_around_bytes = rounddown_pow_of_two(val);
   4410	else
   4411		fault_around_bytes = PAGE_SIZE; /* rounddown_pow_of_two(0) is undefined */
   4412	return 0;
   4413}
   4414DEFINE_DEBUGFS_ATTRIBUTE(fault_around_bytes_fops,
   4415		fault_around_bytes_get, fault_around_bytes_set, "%llu\n");
   4416
   4417static int __init fault_around_debugfs(void)
   4418{
   4419	debugfs_create_file_unsafe("fault_around_bytes", 0644, NULL, NULL,
   4420				   &fault_around_bytes_fops);
   4421	return 0;
   4422}
   4423late_initcall(fault_around_debugfs);
   4424#endif
   4425
   4426/*
   4427 * do_fault_around() tries to map few pages around the fault address. The hope
   4428 * is that the pages will be needed soon and this will lower the number of
   4429 * faults to handle.
   4430 *
   4431 * It uses vm_ops->map_pages() to map the pages, which skips the page if it's
   4432 * not ready to be mapped: not up-to-date, locked, etc.
   4433 *
   4434 * This function is called with the page table lock taken. In the split ptlock
   4435 * case the page table lock only protects only those entries which belong to
   4436 * the page table corresponding to the fault address.
   4437 *
   4438 * This function doesn't cross the VMA boundaries, in order to call map_pages()
   4439 * only once.
   4440 *
   4441 * fault_around_bytes defines how many bytes we'll try to map.
   4442 * do_fault_around() expects it to be set to a power of two less than or equal
   4443 * to PTRS_PER_PTE.
   4444 *
   4445 * The virtual address of the area that we map is naturally aligned to
   4446 * fault_around_bytes rounded down to the machine page size
   4447 * (and therefore to page order).  This way it's easier to guarantee
   4448 * that we don't cross page table boundaries.
   4449 */
   4450static vm_fault_t do_fault_around(struct vm_fault *vmf)
   4451{
   4452	unsigned long address = vmf->address, nr_pages, mask;
   4453	pgoff_t start_pgoff = vmf->pgoff;
   4454	pgoff_t end_pgoff;
   4455	int off;
   4456
   4457	nr_pages = READ_ONCE(fault_around_bytes) >> PAGE_SHIFT;
   4458	mask = ~(nr_pages * PAGE_SIZE - 1) & PAGE_MASK;
   4459
   4460	address = max(address & mask, vmf->vma->vm_start);
   4461	off = ((vmf->address - address) >> PAGE_SHIFT) & (PTRS_PER_PTE - 1);
   4462	start_pgoff -= off;
   4463
   4464	/*
   4465	 *  end_pgoff is either the end of the page table, the end of
   4466	 *  the vma or nr_pages from start_pgoff, depending what is nearest.
   4467	 */
   4468	end_pgoff = start_pgoff -
   4469		((address >> PAGE_SHIFT) & (PTRS_PER_PTE - 1)) +
   4470		PTRS_PER_PTE - 1;
   4471	end_pgoff = min3(end_pgoff, vma_pages(vmf->vma) + vmf->vma->vm_pgoff - 1,
   4472			start_pgoff + nr_pages - 1);
   4473
   4474	if (pmd_none(*vmf->pmd)) {
   4475		vmf->prealloc_pte = pte_alloc_one(vmf->vma->vm_mm);
   4476		if (!vmf->prealloc_pte)
   4477			return VM_FAULT_OOM;
   4478	}
   4479
   4480	return vmf->vma->vm_ops->map_pages(vmf, start_pgoff, end_pgoff);
   4481}
   4482
   4483/* Return true if we should do read fault-around, false otherwise */
   4484static inline bool should_fault_around(struct vm_fault *vmf)
   4485{
   4486	/* No ->map_pages?  No way to fault around... */
   4487	if (!vmf->vma->vm_ops->map_pages)
   4488		return false;
   4489
   4490	if (uffd_disable_fault_around(vmf->vma))
   4491		return false;
   4492
   4493	return fault_around_bytes >> PAGE_SHIFT > 1;
   4494}
   4495
   4496static vm_fault_t do_read_fault(struct vm_fault *vmf)
   4497{
   4498	vm_fault_t ret = 0;
   4499
   4500	/*
   4501	 * Let's call ->map_pages() first and use ->fault() as fallback
   4502	 * if page by the offset is not ready to be mapped (cold cache or
   4503	 * something).
   4504	 */
   4505	if (should_fault_around(vmf)) {
   4506		ret = do_fault_around(vmf);
   4507		if (ret)
   4508			return ret;
   4509	}
   4510
   4511	ret = __do_fault(vmf);
   4512	if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
   4513		return ret;
   4514
   4515	ret |= finish_fault(vmf);
   4516	unlock_page(vmf->page);
   4517	if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
   4518		put_page(vmf->page);
   4519	return ret;
   4520}
   4521
   4522static vm_fault_t do_cow_fault(struct vm_fault *vmf)
   4523{
   4524	struct vm_area_struct *vma = vmf->vma;
   4525	vm_fault_t ret;
   4526
   4527	if (unlikely(anon_vma_prepare(vma)))
   4528		return VM_FAULT_OOM;
   4529
   4530	vmf->cow_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, vmf->address);
   4531	if (!vmf->cow_page)
   4532		return VM_FAULT_OOM;
   4533
   4534	if (mem_cgroup_charge(page_folio(vmf->cow_page), vma->vm_mm,
   4535				GFP_KERNEL)) {
   4536		put_page(vmf->cow_page);
   4537		return VM_FAULT_OOM;
   4538	}
   4539	cgroup_throttle_swaprate(vmf->cow_page, GFP_KERNEL);
   4540
   4541	ret = __do_fault(vmf);
   4542	if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
   4543		goto uncharge_out;
   4544	if (ret & VM_FAULT_DONE_COW)
   4545		return ret;
   4546
   4547	copy_user_highpage(vmf->cow_page, vmf->page, vmf->address, vma);
   4548	__SetPageUptodate(vmf->cow_page);
   4549
   4550	ret |= finish_fault(vmf);
   4551	unlock_page(vmf->page);
   4552	put_page(vmf->page);
   4553	if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
   4554		goto uncharge_out;
   4555	return ret;
   4556uncharge_out:
   4557	put_page(vmf->cow_page);
   4558	return ret;
   4559}
   4560
   4561static vm_fault_t do_shared_fault(struct vm_fault *vmf)
   4562{
   4563	struct vm_area_struct *vma = vmf->vma;
   4564	vm_fault_t ret, tmp;
   4565
   4566	ret = __do_fault(vmf);
   4567	if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
   4568		return ret;
   4569
   4570	/*
   4571	 * Check if the backing address space wants to know that the page is
   4572	 * about to become writable
   4573	 */
   4574	if (vma->vm_ops->page_mkwrite) {
   4575		unlock_page(vmf->page);
   4576		tmp = do_page_mkwrite(vmf);
   4577		if (unlikely(!tmp ||
   4578				(tmp & (VM_FAULT_ERROR | VM_FAULT_NOPAGE)))) {
   4579			put_page(vmf->page);
   4580			return tmp;
   4581		}
   4582	}
   4583
   4584	ret |= finish_fault(vmf);
   4585	if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE |
   4586					VM_FAULT_RETRY))) {
   4587		unlock_page(vmf->page);
   4588		put_page(vmf->page);
   4589		return ret;
   4590	}
   4591
   4592	ret |= fault_dirty_shared_page(vmf);
   4593	return ret;
   4594}
   4595
   4596/*
   4597 * We enter with non-exclusive mmap_lock (to exclude vma changes,
   4598 * but allow concurrent faults).
   4599 * The mmap_lock may have been released depending on flags and our
   4600 * return value.  See filemap_fault() and __folio_lock_or_retry().
   4601 * If mmap_lock is released, vma may become invalid (for example
   4602 * by other thread calling munmap()).
   4603 */
   4604static vm_fault_t do_fault(struct vm_fault *vmf)
   4605{
   4606	struct vm_area_struct *vma = vmf->vma;
   4607	struct mm_struct *vm_mm = vma->vm_mm;
   4608	vm_fault_t ret;
   4609
   4610	/*
   4611	 * The VMA was not fully populated on mmap() or missing VM_DONTEXPAND
   4612	 */
   4613	if (!vma->vm_ops->fault) {
   4614		/*
   4615		 * If we find a migration pmd entry or a none pmd entry, which
   4616		 * should never happen, return SIGBUS
   4617		 */
   4618		if (unlikely(!pmd_present(*vmf->pmd)))
   4619			ret = VM_FAULT_SIGBUS;
   4620		else {
   4621			vmf->pte = pte_offset_map_lock(vmf->vma->vm_mm,
   4622						       vmf->pmd,
   4623						       vmf->address,
   4624						       &vmf->ptl);
   4625			/*
   4626			 * Make sure this is not a temporary clearing of pte
   4627			 * by holding ptl and checking again. A R/M/W update
   4628			 * of pte involves: take ptl, clearing the pte so that
   4629			 * we don't have concurrent modification by hardware
   4630			 * followed by an update.
   4631			 */
   4632			if (unlikely(pte_none(*vmf->pte)))
   4633				ret = VM_FAULT_SIGBUS;
   4634			else
   4635				ret = VM_FAULT_NOPAGE;
   4636
   4637			pte_unmap_unlock(vmf->pte, vmf->ptl);
   4638		}
   4639	} else if (!(vmf->flags & FAULT_FLAG_WRITE))
   4640		ret = do_read_fault(vmf);
   4641	else if (!(vma->vm_flags & VM_SHARED))
   4642		ret = do_cow_fault(vmf);
   4643	else
   4644		ret = do_shared_fault(vmf);
   4645
   4646	/* preallocated pagetable is unused: free it */
   4647	if (vmf->prealloc_pte) {
   4648		pte_free(vm_mm, vmf->prealloc_pte);
   4649		vmf->prealloc_pte = NULL;
   4650	}
   4651	return ret;
   4652}
   4653
   4654int numa_migrate_prep(struct page *page, struct vm_area_struct *vma,
   4655		      unsigned long addr, int page_nid, int *flags)
   4656{
   4657	get_page(page);
   4658
   4659	count_vm_numa_event(NUMA_HINT_FAULTS);
   4660	if (page_nid == numa_node_id()) {
   4661		count_vm_numa_event(NUMA_HINT_FAULTS_LOCAL);
   4662		*flags |= TNF_FAULT_LOCAL;
   4663	}
   4664
   4665	return mpol_misplaced(page, vma, addr);
   4666}
   4667
   4668static vm_fault_t do_numa_page(struct vm_fault *vmf)
   4669{
   4670	struct vm_area_struct *vma = vmf->vma;
   4671	struct page *page = NULL;
   4672	int page_nid = NUMA_NO_NODE;
   4673	int last_cpupid;
   4674	int target_nid;
   4675	pte_t pte, old_pte;
   4676	bool was_writable = pte_savedwrite(vmf->orig_pte);
   4677	int flags = 0;
   4678
   4679	/*
   4680	 * The "pte" at this point cannot be used safely without
   4681	 * validation through pte_unmap_same(). It's of NUMA type but
   4682	 * the pfn may be screwed if the read is non atomic.
   4683	 */
   4684	vmf->ptl = pte_lockptr(vma->vm_mm, vmf->pmd);
   4685	spin_lock(vmf->ptl);
   4686	if (unlikely(!pte_same(*vmf->pte, vmf->orig_pte))) {
   4687		pte_unmap_unlock(vmf->pte, vmf->ptl);
   4688		goto out;
   4689	}
   4690
   4691	/* Get the normal PTE  */
   4692	old_pte = ptep_get(vmf->pte);
   4693	pte = pte_modify(old_pte, vma->vm_page_prot);
   4694
   4695	page = vm_normal_page(vma, vmf->address, pte);
   4696	if (!page)
   4697		goto out_map;
   4698
   4699	/* TODO: handle PTE-mapped THP */
   4700	if (PageCompound(page))
   4701		goto out_map;
   4702
   4703	/*
   4704	 * Avoid grouping on RO pages in general. RO pages shouldn't hurt as
   4705	 * much anyway since they can be in shared cache state. This misses
   4706	 * the case where a mapping is writable but the process never writes
   4707	 * to it but pte_write gets cleared during protection updates and
   4708	 * pte_dirty has unpredictable behaviour between PTE scan updates,
   4709	 * background writeback, dirty balancing and application behaviour.
   4710	 */
   4711	if (!was_writable)
   4712		flags |= TNF_NO_GROUP;
   4713
   4714	/*
   4715	 * Flag if the page is shared between multiple address spaces. This
   4716	 * is later used when determining whether to group tasks together
   4717	 */
   4718	if (page_mapcount(page) > 1 && (vma->vm_flags & VM_SHARED))
   4719		flags |= TNF_SHARED;
   4720
   4721	last_cpupid = page_cpupid_last(page);
   4722	page_nid = page_to_nid(page);
   4723	target_nid = numa_migrate_prep(page, vma, vmf->address, page_nid,
   4724			&flags);
   4725	if (target_nid == NUMA_NO_NODE) {
   4726		put_page(page);
   4727		goto out_map;
   4728	}
   4729	pte_unmap_unlock(vmf->pte, vmf->ptl);
   4730
   4731	/* Migrate to the requested node */
   4732	if (migrate_misplaced_page(page, vma, target_nid)) {
   4733		page_nid = target_nid;
   4734		flags |= TNF_MIGRATED;
   4735	} else {
   4736		flags |= TNF_MIGRATE_FAIL;
   4737		vmf->pte = pte_offset_map(vmf->pmd, vmf->address);
   4738		spin_lock(vmf->ptl);
   4739		if (unlikely(!pte_same(*vmf->pte, vmf->orig_pte))) {
   4740			pte_unmap_unlock(vmf->pte, vmf->ptl);
   4741			goto out;
   4742		}
   4743		goto out_map;
   4744	}
   4745
   4746out:
   4747	if (page_nid != NUMA_NO_NODE)
   4748		task_numa_fault(last_cpupid, page_nid, 1, flags);
   4749	return 0;
   4750out_map:
   4751	/*
   4752	 * Make it present again, depending on how arch implements
   4753	 * non-accessible ptes, some can allow access by kernel mode.
   4754	 */
   4755	old_pte = ptep_modify_prot_start(vma, vmf->address, vmf->pte);
   4756	pte = pte_modify(old_pte, vma->vm_page_prot);
   4757	pte = pte_mkyoung(pte);
   4758	if (was_writable)
   4759		pte = pte_mkwrite(pte);
   4760	ptep_modify_prot_commit(vma, vmf->address, vmf->pte, old_pte, pte);
   4761	update_mmu_cache(vma, vmf->address, vmf->pte);
   4762	pte_unmap_unlock(vmf->pte, vmf->ptl);
   4763	goto out;
   4764}
   4765
   4766static inline vm_fault_t create_huge_pmd(struct vm_fault *vmf)
   4767{
   4768	if (vma_is_anonymous(vmf->vma))
   4769		return do_huge_pmd_anonymous_page(vmf);
   4770	if (vmf->vma->vm_ops->huge_fault)
   4771		return vmf->vma->vm_ops->huge_fault(vmf, PE_SIZE_PMD);
   4772	return VM_FAULT_FALLBACK;
   4773}
   4774
   4775/* `inline' is required to avoid gcc 4.1.2 build error */
   4776static inline vm_fault_t wp_huge_pmd(struct vm_fault *vmf)
   4777{
   4778	const bool unshare = vmf->flags & FAULT_FLAG_UNSHARE;
   4779
   4780	if (vma_is_anonymous(vmf->vma)) {
   4781		if (likely(!unshare) &&
   4782		    userfaultfd_huge_pmd_wp(vmf->vma, vmf->orig_pmd))
   4783			return handle_userfault(vmf, VM_UFFD_WP);
   4784		return do_huge_pmd_wp_page(vmf);
   4785	}
   4786	if (vmf->vma->vm_ops->huge_fault) {
   4787		vm_fault_t ret = vmf->vma->vm_ops->huge_fault(vmf, PE_SIZE_PMD);
   4788
   4789		if (!(ret & VM_FAULT_FALLBACK))
   4790			return ret;
   4791	}
   4792
   4793	/* COW or write-notify handled on pte level: split pmd. */
   4794	__split_huge_pmd(vmf->vma, vmf->pmd, vmf->address, false, NULL);
   4795
   4796	return VM_FAULT_FALLBACK;
   4797}
   4798
   4799static vm_fault_t create_huge_pud(struct vm_fault *vmf)
   4800{
   4801#if defined(CONFIG_TRANSPARENT_HUGEPAGE) &&			\
   4802	defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD)
   4803	/* No support for anonymous transparent PUD pages yet */
   4804	if (vma_is_anonymous(vmf->vma))
   4805		goto split;
   4806	if (vmf->vma->vm_ops->huge_fault) {
   4807		vm_fault_t ret = vmf->vma->vm_ops->huge_fault(vmf, PE_SIZE_PUD);
   4808
   4809		if (!(ret & VM_FAULT_FALLBACK))
   4810			return ret;
   4811	}
   4812split:
   4813	/* COW or write-notify not handled on PUD level: split pud.*/
   4814	__split_huge_pud(vmf->vma, vmf->pud, vmf->address);
   4815#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
   4816	return VM_FAULT_FALLBACK;
   4817}
   4818
   4819static vm_fault_t wp_huge_pud(struct vm_fault *vmf, pud_t orig_pud)
   4820{
   4821#ifdef CONFIG_TRANSPARENT_HUGEPAGE
   4822	/* No support for anonymous transparent PUD pages yet */
   4823	if (vma_is_anonymous(vmf->vma))
   4824		return VM_FAULT_FALLBACK;
   4825	if (vmf->vma->vm_ops->huge_fault)
   4826		return vmf->vma->vm_ops->huge_fault(vmf, PE_SIZE_PUD);
   4827#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
   4828	return VM_FAULT_FALLBACK;
   4829}
   4830
   4831/*
   4832 * These routines also need to handle stuff like marking pages dirty
   4833 * and/or accessed for architectures that don't do it in hardware (most
   4834 * RISC architectures).  The early dirtying is also good on the i386.
   4835 *
   4836 * There is also a hook called "update_mmu_cache()" that architectures
   4837 * with external mmu caches can use to update those (ie the Sparc or
   4838 * PowerPC hashed page tables that act as extended TLBs).
   4839 *
   4840 * We enter with non-exclusive mmap_lock (to exclude vma changes, but allow
   4841 * concurrent faults).
   4842 *
   4843 * The mmap_lock may have been released depending on flags and our return value.
   4844 * See filemap_fault() and __folio_lock_or_retry().
   4845 */
   4846static vm_fault_t handle_pte_fault(struct vm_fault *vmf)
   4847{
   4848	pte_t entry;
   4849
   4850	if (unlikely(pmd_none(*vmf->pmd))) {
   4851		/*
   4852		 * Leave __pte_alloc() until later: because vm_ops->fault may
   4853		 * want to allocate huge page, and if we expose page table
   4854		 * for an instant, it will be difficult to retract from
   4855		 * concurrent faults and from rmap lookups.
   4856		 */
   4857		vmf->pte = NULL;
   4858		vmf->flags &= ~FAULT_FLAG_ORIG_PTE_VALID;
   4859	} else {
   4860		/*
   4861		 * If a huge pmd materialized under us just retry later.  Use
   4862		 * pmd_trans_unstable() via pmd_devmap_trans_unstable() instead
   4863		 * of pmd_trans_huge() to ensure the pmd didn't become
   4864		 * pmd_trans_huge under us and then back to pmd_none, as a
   4865		 * result of MADV_DONTNEED running immediately after a huge pmd
   4866		 * fault in a different thread of this mm, in turn leading to a
   4867		 * misleading pmd_trans_huge() retval. All we have to ensure is
   4868		 * that it is a regular pmd that we can walk with
   4869		 * pte_offset_map() and we can do that through an atomic read
   4870		 * in C, which is what pmd_trans_unstable() provides.
   4871		 */
   4872		if (pmd_devmap_trans_unstable(vmf->pmd))
   4873			return 0;
   4874		/*
   4875		 * A regular pmd is established and it can't morph into a huge
   4876		 * pmd from under us anymore at this point because we hold the
   4877		 * mmap_lock read mode and khugepaged takes it in write mode.
   4878		 * So now it's safe to run pte_offset_map().
   4879		 */
   4880		vmf->pte = pte_offset_map(vmf->pmd, vmf->address);
   4881		vmf->orig_pte = *vmf->pte;
   4882		vmf->flags |= FAULT_FLAG_ORIG_PTE_VALID;
   4883
   4884		/*
   4885		 * some architectures can have larger ptes than wordsize,
   4886		 * e.g.ppc44x-defconfig has CONFIG_PTE_64BIT=y and
   4887		 * CONFIG_32BIT=y, so READ_ONCE cannot guarantee atomic
   4888		 * accesses.  The code below just needs a consistent view
   4889		 * for the ifs and we later double check anyway with the
   4890		 * ptl lock held. So here a barrier will do.
   4891		 */
   4892		barrier();
   4893		if (pte_none(vmf->orig_pte)) {
   4894			pte_unmap(vmf->pte);
   4895			vmf->pte = NULL;
   4896		}
   4897	}
   4898
   4899	if (!vmf->pte) {
   4900		if (vma_is_anonymous(vmf->vma))
   4901			return do_anonymous_page(vmf);
   4902		else
   4903			return do_fault(vmf);
   4904	}
   4905
   4906	if (!pte_present(vmf->orig_pte))
   4907		return do_swap_page(vmf);
   4908
   4909	if (pte_protnone(vmf->orig_pte) && vma_is_accessible(vmf->vma))
   4910		return do_numa_page(vmf);
   4911
   4912	vmf->ptl = pte_lockptr(vmf->vma->vm_mm, vmf->pmd);
   4913	spin_lock(vmf->ptl);
   4914	entry = vmf->orig_pte;
   4915	if (unlikely(!pte_same(*vmf->pte, entry))) {
   4916		update_mmu_tlb(vmf->vma, vmf->address, vmf->pte);
   4917		goto unlock;
   4918	}
   4919	if (vmf->flags & (FAULT_FLAG_WRITE|FAULT_FLAG_UNSHARE)) {
   4920		if (!pte_write(entry))
   4921			return do_wp_page(vmf);
   4922		else if (likely(vmf->flags & FAULT_FLAG_WRITE))
   4923			entry = pte_mkdirty(entry);
   4924	}
   4925	entry = pte_mkyoung(entry);
   4926	if (ptep_set_access_flags(vmf->vma, vmf->address, vmf->pte, entry,
   4927				vmf->flags & FAULT_FLAG_WRITE)) {
   4928		update_mmu_cache(vmf->vma, vmf->address, vmf->pte);
   4929	} else {
   4930		/* Skip spurious TLB flush for retried page fault */
   4931		if (vmf->flags & FAULT_FLAG_TRIED)
   4932			goto unlock;
   4933		/*
   4934		 * This is needed only for protection faults but the arch code
   4935		 * is not yet telling us if this is a protection fault or not.
   4936		 * This still avoids useless tlb flushes for .text page faults
   4937		 * with threads.
   4938		 */
   4939		if (vmf->flags & FAULT_FLAG_WRITE)
   4940			flush_tlb_fix_spurious_fault(vmf->vma, vmf->address);
   4941	}
   4942unlock:
   4943	pte_unmap_unlock(vmf->pte, vmf->ptl);
   4944	return 0;
   4945}
   4946
   4947static int handle_split_page_fault(struct vm_fault *vmf)
   4948{
   4949	if (!IS_ENABLED(CONFIG_AMD_MEM_ENCRYPT))
   4950		return VM_FAULT_SIGBUS;
   4951
   4952	__split_huge_pmd(vmf->vma, vmf->pmd, vmf->address, false, NULL);
   4953	return 0;
   4954}
   4955
   4956/*
   4957 * By the time we get here, we already hold the mm semaphore
   4958 *
   4959 * The mmap_lock may have been released depending on flags and our
   4960 * return value.  See filemap_fault() and __folio_lock_or_retry().
   4961 */
   4962static vm_fault_t __handle_mm_fault(struct vm_area_struct *vma,
   4963		unsigned long address, unsigned int flags)
   4964{
   4965	struct vm_fault vmf = {
   4966		.vma = vma,
   4967		.address = address & PAGE_MASK,
   4968		.real_address = address,
   4969		.flags = flags,
   4970		.pgoff = linear_page_index(vma, address),
   4971		.gfp_mask = __get_fault_gfp_mask(vma),
   4972	};
   4973	struct mm_struct *mm = vma->vm_mm;
   4974	pgd_t *pgd;
   4975	p4d_t *p4d;
   4976	vm_fault_t ret;
   4977
   4978	pgd = pgd_offset(mm, address);
   4979	p4d = p4d_alloc(mm, pgd, address);
   4980	if (!p4d)
   4981		return VM_FAULT_OOM;
   4982
   4983	vmf.pud = pud_alloc(mm, p4d, address);
   4984	if (!vmf.pud)
   4985		return VM_FAULT_OOM;
   4986retry_pud:
   4987	if (pud_none(*vmf.pud) && __transparent_hugepage_enabled(vma)) {
   4988		ret = create_huge_pud(&vmf);
   4989		if (!(ret & VM_FAULT_FALLBACK))
   4990			return ret;
   4991	} else {
   4992		pud_t orig_pud = *vmf.pud;
   4993
   4994		barrier();
   4995		if (pud_trans_huge(orig_pud) || pud_devmap(orig_pud)) {
   4996
   4997			/*
   4998			 * TODO once we support anonymous PUDs: NUMA case and
   4999			 * FAULT_FLAG_UNSHARE handling.
   5000			 */
   5001			if ((flags & FAULT_FLAG_WRITE) && !pud_write(orig_pud)) {
   5002				ret = wp_huge_pud(&vmf, orig_pud);
   5003				if (!(ret & VM_FAULT_FALLBACK))
   5004					return ret;
   5005			} else {
   5006				huge_pud_set_accessed(&vmf, orig_pud);
   5007				return 0;
   5008			}
   5009		}
   5010	}
   5011
   5012	vmf.pmd = pmd_alloc(mm, vmf.pud, address);
   5013	if (!vmf.pmd)
   5014		return VM_FAULT_OOM;
   5015
   5016	/* Huge pud page fault raced with pmd_alloc? */
   5017	if (pud_trans_unstable(vmf.pud))
   5018		goto retry_pud;
   5019
   5020	if (pmd_none(*vmf.pmd) && __transparent_hugepage_enabled(vma)) {
   5021		ret = create_huge_pmd(&vmf);
   5022		if (!(ret & VM_FAULT_FALLBACK))
   5023			return ret;
   5024	} else {
   5025		vmf.orig_pmd = *vmf.pmd;
   5026
   5027		barrier();
   5028		if (unlikely(is_swap_pmd(vmf.orig_pmd))) {
   5029			VM_BUG_ON(thp_migration_supported() &&
   5030					  !is_pmd_migration_entry(vmf.orig_pmd));
   5031			if (is_pmd_migration_entry(vmf.orig_pmd))
   5032				pmd_migration_entry_wait(mm, vmf.pmd);
   5033			return 0;
   5034		}
   5035
   5036		if (flags & FAULT_FLAG_PAGE_SPLIT)
   5037			return handle_split_page_fault(&vmf);
   5038
   5039		if (pmd_trans_huge(vmf.orig_pmd) || pmd_devmap(vmf.orig_pmd)) {
   5040			if (pmd_protnone(vmf.orig_pmd) && vma_is_accessible(vma))
   5041				return do_huge_pmd_numa_page(&vmf);
   5042
   5043			if ((flags & (FAULT_FLAG_WRITE|FAULT_FLAG_UNSHARE)) &&
   5044			    !pmd_write(vmf.orig_pmd)) {
   5045				ret = wp_huge_pmd(&vmf);
   5046				if (!(ret & VM_FAULT_FALLBACK))
   5047					return ret;
   5048			} else {
   5049				huge_pmd_set_accessed(&vmf);
   5050				return 0;
   5051			}
   5052		}
   5053	}
   5054
   5055	return handle_pte_fault(&vmf);
   5056}
   5057
   5058/**
   5059 * mm_account_fault - Do page fault accounting
   5060 *
   5061 * @regs: the pt_regs struct pointer.  When set to NULL, will skip accounting
   5062 *        of perf event counters, but we'll still do the per-task accounting to
   5063 *        the task who triggered this page fault.
   5064 * @address: the faulted address.
   5065 * @flags: the fault flags.
   5066 * @ret: the fault retcode.
   5067 *
   5068 * This will take care of most of the page fault accounting.  Meanwhile, it
   5069 * will also include the PERF_COUNT_SW_PAGE_FAULTS_[MAJ|MIN] perf counter
   5070 * updates.  However, note that the handling of PERF_COUNT_SW_PAGE_FAULTS should
   5071 * still be in per-arch page fault handlers at the entry of page fault.
   5072 */
   5073static inline void mm_account_fault(struct pt_regs *regs,
   5074				    unsigned long address, unsigned int flags,
   5075				    vm_fault_t ret)
   5076{
   5077	bool major;
   5078
   5079	/*
   5080	 * We don't do accounting for some specific faults:
   5081	 *
   5082	 * - Unsuccessful faults (e.g. when the address wasn't valid).  That
   5083	 *   includes arch_vma_access_permitted() failing before reaching here.
   5084	 *   So this is not a "this many hardware page faults" counter.  We
   5085	 *   should use the hw profiling for that.
   5086	 *
   5087	 * - Incomplete faults (VM_FAULT_RETRY).  They will only be counted
   5088	 *   once they're completed.
   5089	 */
   5090	if (ret & (VM_FAULT_ERROR | VM_FAULT_RETRY))
   5091		return;
   5092
   5093	/*
   5094	 * We define the fault as a major fault when the final successful fault
   5095	 * is VM_FAULT_MAJOR, or if it retried (which implies that we couldn't
   5096	 * handle it immediately previously).
   5097	 */
   5098	major = (ret & VM_FAULT_MAJOR) || (flags & FAULT_FLAG_TRIED);
   5099
   5100	if (major)
   5101		current->maj_flt++;
   5102	else
   5103		current->min_flt++;
   5104
   5105	/*
   5106	 * If the fault is done for GUP, regs will be NULL.  We only do the
   5107	 * accounting for the per thread fault counters who triggered the
   5108	 * fault, and we skip the perf event updates.
   5109	 */
   5110	if (!regs)
   5111		return;
   5112
   5113	if (major)
   5114		perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, regs, address);
   5115	else
   5116		perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, regs, address);
   5117}
   5118
   5119/*
   5120 * By the time we get here, we already hold the mm semaphore
   5121 *
   5122 * The mmap_lock may have been released depending on flags and our
   5123 * return value.  See filemap_fault() and __folio_lock_or_retry().
   5124 */
   5125vm_fault_t handle_mm_fault(struct vm_area_struct *vma, unsigned long address,
   5126			   unsigned int flags, struct pt_regs *regs)
   5127{
   5128	vm_fault_t ret;
   5129
   5130	__set_current_state(TASK_RUNNING);
   5131
   5132	count_vm_event(PGFAULT);
   5133	count_memcg_event_mm(vma->vm_mm, PGFAULT);
   5134
   5135	/* do counter updates before entering really critical section. */
   5136	check_sync_rss_stat(current);
   5137
   5138	if (!arch_vma_access_permitted(vma, flags & FAULT_FLAG_WRITE,
   5139					    flags & FAULT_FLAG_INSTRUCTION,
   5140					    flags & FAULT_FLAG_REMOTE))
   5141		return VM_FAULT_SIGSEGV;
   5142
   5143	/*
   5144	 * Enable the memcg OOM handling for faults triggered in user
   5145	 * space.  Kernel faults are handled more gracefully.
   5146	 */
   5147	if (flags & FAULT_FLAG_USER)
   5148		mem_cgroup_enter_user_fault();
   5149
   5150	if (unlikely(is_vm_hugetlb_page(vma)))
   5151		ret = hugetlb_fault(vma->vm_mm, vma, address, flags);
   5152	else
   5153		ret = __handle_mm_fault(vma, address, flags);
   5154
   5155	if (flags & FAULT_FLAG_USER) {
   5156		mem_cgroup_exit_user_fault();
   5157		/*
   5158		 * The task may have entered a memcg OOM situation but
   5159		 * if the allocation error was handled gracefully (no
   5160		 * VM_FAULT_OOM), there is no need to kill anything.
   5161		 * Just clean up the OOM state peacefully.
   5162		 */
   5163		if (task_in_memcg_oom(current) && !(ret & VM_FAULT_OOM))
   5164			mem_cgroup_oom_synchronize(false);
   5165	}
   5166
   5167	mm_account_fault(regs, address, flags, ret);
   5168
   5169	return ret;
   5170}
   5171EXPORT_SYMBOL_GPL(handle_mm_fault);
   5172
   5173#ifndef __PAGETABLE_P4D_FOLDED
   5174/*
   5175 * Allocate p4d page table.
   5176 * We've already handled the fast-path in-line.
   5177 */
   5178int __p4d_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address)
   5179{
   5180	p4d_t *new = p4d_alloc_one(mm, address);
   5181	if (!new)
   5182		return -ENOMEM;
   5183
   5184	spin_lock(&mm->page_table_lock);
   5185	if (pgd_present(*pgd)) {	/* Another has populated it */
   5186		p4d_free(mm, new);
   5187	} else {
   5188		smp_wmb(); /* See comment in pmd_install() */
   5189		pgd_populate(mm, pgd, new);
   5190	}
   5191	spin_unlock(&mm->page_table_lock);
   5192	return 0;
   5193}
   5194#endif /* __PAGETABLE_P4D_FOLDED */
   5195
   5196#ifndef __PAGETABLE_PUD_FOLDED
   5197/*
   5198 * Allocate page upper directory.
   5199 * We've already handled the fast-path in-line.
   5200 */
   5201int __pud_alloc(struct mm_struct *mm, p4d_t *p4d, unsigned long address)
   5202{
   5203	pud_t *new = pud_alloc_one(mm, address);
   5204	if (!new)
   5205		return -ENOMEM;
   5206
   5207	spin_lock(&mm->page_table_lock);
   5208	if (!p4d_present(*p4d)) {
   5209		mm_inc_nr_puds(mm);
   5210		smp_wmb(); /* See comment in pmd_install() */
   5211		p4d_populate(mm, p4d, new);
   5212	} else	/* Another has populated it */
   5213		pud_free(mm, new);
   5214	spin_unlock(&mm->page_table_lock);
   5215	return 0;
   5216}
   5217#endif /* __PAGETABLE_PUD_FOLDED */
   5218
   5219#ifndef __PAGETABLE_PMD_FOLDED
   5220/*
   5221 * Allocate page middle directory.
   5222 * We've already handled the fast-path in-line.
   5223 */
   5224int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address)
   5225{
   5226	spinlock_t *ptl;
   5227	pmd_t *new = pmd_alloc_one(mm, address);
   5228	if (!new)
   5229		return -ENOMEM;
   5230
   5231	ptl = pud_lock(mm, pud);
   5232	if (!pud_present(*pud)) {
   5233		mm_inc_nr_pmds(mm);
   5234		smp_wmb(); /* See comment in pmd_install() */
   5235		pud_populate(mm, pud, new);
   5236	} else {	/* Another has populated it */
   5237		pmd_free(mm, new);
   5238	}
   5239	spin_unlock(ptl);
   5240	return 0;
   5241}
   5242#endif /* __PAGETABLE_PMD_FOLDED */
   5243
   5244/**
   5245 * follow_pte - look up PTE at a user virtual address
   5246 * @mm: the mm_struct of the target address space
   5247 * @address: user virtual address
   5248 * @ptepp: location to store found PTE
   5249 * @ptlp: location to store the lock for the PTE
   5250 *
   5251 * On a successful return, the pointer to the PTE is stored in @ptepp;
   5252 * the corresponding lock is taken and its location is stored in @ptlp.
   5253 * The contents of the PTE are only stable until @ptlp is released;
   5254 * any further use, if any, must be protected against invalidation
   5255 * with MMU notifiers.
   5256 *
   5257 * Only IO mappings and raw PFN mappings are allowed.  The mmap semaphore
   5258 * should be taken for read.
   5259 *
   5260 * KVM uses this function.  While it is arguably less bad than ``follow_pfn``,
   5261 * it is not a good general-purpose API.
   5262 *
   5263 * Return: zero on success, -ve otherwise.
   5264 */
   5265int follow_pte(struct mm_struct *mm, unsigned long address,
   5266	       pte_t **ptepp, spinlock_t **ptlp)
   5267{
   5268	pgd_t *pgd;
   5269	p4d_t *p4d;
   5270	pud_t *pud;
   5271	pmd_t *pmd;
   5272	pte_t *ptep;
   5273
   5274	pgd = pgd_offset(mm, address);
   5275	if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd)))
   5276		goto out;
   5277
   5278	p4d = p4d_offset(pgd, address);
   5279	if (p4d_none(*p4d) || unlikely(p4d_bad(*p4d)))
   5280		goto out;
   5281
   5282	pud = pud_offset(p4d, address);
   5283	if (pud_none(*pud) || unlikely(pud_bad(*pud)))
   5284		goto out;
   5285
   5286	pmd = pmd_offset(pud, address);
   5287	VM_BUG_ON(pmd_trans_huge(*pmd));
   5288
   5289	if (pmd_none(*pmd) || unlikely(pmd_bad(*pmd)))
   5290		goto out;
   5291
   5292	ptep = pte_offset_map_lock(mm, pmd, address, ptlp);
   5293	if (!pte_present(*ptep))
   5294		goto unlock;
   5295	*ptepp = ptep;
   5296	return 0;
   5297unlock:
   5298	pte_unmap_unlock(ptep, *ptlp);
   5299out:
   5300	return -EINVAL;
   5301}
   5302EXPORT_SYMBOL_GPL(follow_pte);
   5303
   5304/**
   5305 * follow_pfn - look up PFN at a user virtual address
   5306 * @vma: memory mapping
   5307 * @address: user virtual address
   5308 * @pfn: location to store found PFN
   5309 *
   5310 * Only IO mappings and raw PFN mappings are allowed.
   5311 *
   5312 * This function does not allow the caller to read the permissions
   5313 * of the PTE.  Do not use it.
   5314 *
   5315 * Return: zero and the pfn at @pfn on success, -ve otherwise.
   5316 */
   5317int follow_pfn(struct vm_area_struct *vma, unsigned long address,
   5318	unsigned long *pfn)
   5319{
   5320	int ret = -EINVAL;
   5321	spinlock_t *ptl;
   5322	pte_t *ptep;
   5323
   5324	if (!(vma->vm_flags & (VM_IO | VM_PFNMAP)))
   5325		return ret;
   5326
   5327	ret = follow_pte(vma->vm_mm, address, &ptep, &ptl);
   5328	if (ret)
   5329		return ret;
   5330	*pfn = pte_pfn(*ptep);
   5331	pte_unmap_unlock(ptep, ptl);
   5332	return 0;
   5333}
   5334EXPORT_SYMBOL(follow_pfn);
   5335
   5336#ifdef CONFIG_HAVE_IOREMAP_PROT
   5337int follow_phys(struct vm_area_struct *vma,
   5338		unsigned long address, unsigned int flags,
   5339		unsigned long *prot, resource_size_t *phys)
   5340{
   5341	int ret = -EINVAL;
   5342	pte_t *ptep, pte;
   5343	spinlock_t *ptl;
   5344
   5345	if (!(vma->vm_flags & (VM_IO | VM_PFNMAP)))
   5346		goto out;
   5347
   5348	if (follow_pte(vma->vm_mm, address, &ptep, &ptl))
   5349		goto out;
   5350	pte = *ptep;
   5351
   5352	if ((flags & FOLL_WRITE) && !pte_write(pte))
   5353		goto unlock;
   5354
   5355	*prot = pgprot_val(pte_pgprot(pte));
   5356	*phys = (resource_size_t)pte_pfn(pte) << PAGE_SHIFT;
   5357
   5358	ret = 0;
   5359unlock:
   5360	pte_unmap_unlock(ptep, ptl);
   5361out:
   5362	return ret;
   5363}
   5364
   5365/**
   5366 * generic_access_phys - generic implementation for iomem mmap access
   5367 * @vma: the vma to access
   5368 * @addr: userspace address, not relative offset within @vma
   5369 * @buf: buffer to read/write
   5370 * @len: length of transfer
   5371 * @write: set to FOLL_WRITE when writing, otherwise reading
   5372 *
   5373 * This is a generic implementation for &vm_operations_struct.access for an
   5374 * iomem mapping. This callback is used by access_process_vm() when the @vma is
   5375 * not page based.
   5376 */
   5377int generic_access_phys(struct vm_area_struct *vma, unsigned long addr,
   5378			void *buf, int len, int write)
   5379{
   5380	resource_size_t phys_addr;
   5381	unsigned long prot = 0;
   5382	void __iomem *maddr;
   5383	pte_t *ptep, pte;
   5384	spinlock_t *ptl;
   5385	int offset = offset_in_page(addr);
   5386	int ret = -EINVAL;
   5387
   5388	if (!(vma->vm_flags & (VM_IO | VM_PFNMAP)))
   5389		return -EINVAL;
   5390
   5391retry:
   5392	if (follow_pte(vma->vm_mm, addr, &ptep, &ptl))
   5393		return -EINVAL;
   5394	pte = *ptep;
   5395	pte_unmap_unlock(ptep, ptl);
   5396
   5397	prot = pgprot_val(pte_pgprot(pte));
   5398	phys_addr = (resource_size_t)pte_pfn(pte) << PAGE_SHIFT;
   5399
   5400	if ((write & FOLL_WRITE) && !pte_write(pte))
   5401		return -EINVAL;
   5402
   5403	maddr = ioremap_prot(phys_addr, PAGE_ALIGN(len + offset), prot);
   5404	if (!maddr)
   5405		return -ENOMEM;
   5406
   5407	if (follow_pte(vma->vm_mm, addr, &ptep, &ptl))
   5408		goto out_unmap;
   5409
   5410	if (!pte_same(pte, *ptep)) {
   5411		pte_unmap_unlock(ptep, ptl);
   5412		iounmap(maddr);
   5413
   5414		goto retry;
   5415	}
   5416
   5417	if (write)
   5418		memcpy_toio(maddr + offset, buf, len);
   5419	else
   5420		memcpy_fromio(buf, maddr + offset, len);
   5421	ret = len;
   5422	pte_unmap_unlock(ptep, ptl);
   5423out_unmap:
   5424	iounmap(maddr);
   5425
   5426	return ret;
   5427}
   5428EXPORT_SYMBOL_GPL(generic_access_phys);
   5429#endif
   5430
   5431/*
   5432 * Access another process' address space as given in mm.
   5433 */
   5434int __access_remote_vm(struct mm_struct *mm, unsigned long addr, void *buf,
   5435		       int len, unsigned int gup_flags)
   5436{
   5437	struct vm_area_struct *vma;
   5438	void *old_buf = buf;
   5439	int write = gup_flags & FOLL_WRITE;
   5440
   5441	if (mmap_read_lock_killable(mm))
   5442		return 0;
   5443
   5444	/* ignore errors, just check how much was successfully transferred */
   5445	while (len) {
   5446		int bytes, ret, offset;
   5447		void *maddr;
   5448		struct page *page = NULL;
   5449
   5450		ret = get_user_pages_remote(mm, addr, 1,
   5451				gup_flags, &page, &vma, NULL);
   5452		if (ret <= 0) {
   5453#ifndef CONFIG_HAVE_IOREMAP_PROT
   5454			break;
   5455#else
   5456			/*
   5457			 * Check if this is a VM_IO | VM_PFNMAP VMA, which
   5458			 * we can access using slightly different code.
   5459			 */
   5460			vma = vma_lookup(mm, addr);
   5461			if (!vma)
   5462				break;
   5463			if (vma->vm_ops && vma->vm_ops->access)
   5464				ret = vma->vm_ops->access(vma, addr, buf,
   5465							  len, write);
   5466			if (ret <= 0)
   5467				break;
   5468			bytes = ret;
   5469#endif
   5470		} else {
   5471			bytes = len;
   5472			offset = addr & (PAGE_SIZE-1);
   5473			if (bytes > PAGE_SIZE-offset)
   5474				bytes = PAGE_SIZE-offset;
   5475
   5476			maddr = kmap(page);
   5477			if (write) {
   5478				copy_to_user_page(vma, page, addr,
   5479						  maddr + offset, buf, bytes);
   5480				set_page_dirty_lock(page);
   5481			} else {
   5482				copy_from_user_page(vma, page, addr,
   5483						    buf, maddr + offset, bytes);
   5484			}
   5485			kunmap(page);
   5486			put_page(page);
   5487		}
   5488		len -= bytes;
   5489		buf += bytes;
   5490		addr += bytes;
   5491	}
   5492	mmap_read_unlock(mm);
   5493
   5494	return buf - old_buf;
   5495}
   5496
   5497/**
   5498 * access_remote_vm - access another process' address space
   5499 * @mm:		the mm_struct of the target address space
   5500 * @addr:	start address to access
   5501 * @buf:	source or destination buffer
   5502 * @len:	number of bytes to transfer
   5503 * @gup_flags:	flags modifying lookup behaviour
   5504 *
   5505 * The caller must hold a reference on @mm.
   5506 *
   5507 * Return: number of bytes copied from source to destination.
   5508 */
   5509int access_remote_vm(struct mm_struct *mm, unsigned long addr,
   5510		void *buf, int len, unsigned int gup_flags)
   5511{
   5512	return __access_remote_vm(mm, addr, buf, len, gup_flags);
   5513}
   5514
   5515/*
   5516 * Access another process' address space.
   5517 * Source/target buffer must be kernel space,
   5518 * Do not walk the page table directly, use get_user_pages
   5519 */
   5520int access_process_vm(struct task_struct *tsk, unsigned long addr,
   5521		void *buf, int len, unsigned int gup_flags)
   5522{
   5523	struct mm_struct *mm;
   5524	int ret;
   5525
   5526	mm = get_task_mm(tsk);
   5527	if (!mm)
   5528		return 0;
   5529
   5530	ret = __access_remote_vm(mm, addr, buf, len, gup_flags);
   5531
   5532	mmput(mm);
   5533
   5534	return ret;
   5535}
   5536EXPORT_SYMBOL_GPL(access_process_vm);
   5537
   5538/*
   5539 * Print the name of a VMA.
   5540 */
   5541void print_vma_addr(char *prefix, unsigned long ip)
   5542{
   5543	struct mm_struct *mm = current->mm;
   5544	struct vm_area_struct *vma;
   5545
   5546	/*
   5547	 * we might be running from an atomic context so we cannot sleep
   5548	 */
   5549	if (!mmap_read_trylock(mm))
   5550		return;
   5551
   5552	vma = find_vma(mm, ip);
   5553	if (vma && vma->vm_file) {
   5554		struct file *f = vma->vm_file;
   5555		char *buf = (char *)__get_free_page(GFP_NOWAIT);
   5556		if (buf) {
   5557			char *p;
   5558
   5559			p = file_path(f, buf, PAGE_SIZE);
   5560			if (IS_ERR(p))
   5561				p = "?";
   5562			printk("%s%s[%lx+%lx]", prefix, kbasename(p),
   5563					vma->vm_start,
   5564					vma->vm_end - vma->vm_start);
   5565			free_page((unsigned long)buf);
   5566		}
   5567	}
   5568	mmap_read_unlock(mm);
   5569}
   5570
   5571#if defined(CONFIG_PROVE_LOCKING) || defined(CONFIG_DEBUG_ATOMIC_SLEEP)
   5572void __might_fault(const char *file, int line)
   5573{
   5574	if (pagefault_disabled())
   5575		return;
   5576	__might_sleep(file, line);
   5577#if defined(CONFIG_DEBUG_ATOMIC_SLEEP)
   5578	if (current->mm)
   5579		might_lock_read(&current->mm->mmap_lock);
   5580#endif
   5581}
   5582EXPORT_SYMBOL(__might_fault);
   5583#endif
   5584
   5585#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLBFS)
   5586/*
   5587 * Process all subpages of the specified huge page with the specified
   5588 * operation.  The target subpage will be processed last to keep its
   5589 * cache lines hot.
   5590 */
   5591static inline void process_huge_page(
   5592	unsigned long addr_hint, unsigned int pages_per_huge_page,
   5593	void (*process_subpage)(unsigned long addr, int idx, void *arg),
   5594	void *arg)
   5595{
   5596	int i, n, base, l;
   5597	unsigned long addr = addr_hint &
   5598		~(((unsigned long)pages_per_huge_page << PAGE_SHIFT) - 1);
   5599
   5600	/* Process target subpage last to keep its cache lines hot */
   5601	might_sleep();
   5602	n = (addr_hint - addr) / PAGE_SIZE;
   5603	if (2 * n <= pages_per_huge_page) {
   5604		/* If target subpage in first half of huge page */
   5605		base = 0;
   5606		l = n;
   5607		/* Process subpages at the end of huge page */
   5608		for (i = pages_per_huge_page - 1; i >= 2 * n; i--) {
   5609			cond_resched();
   5610			process_subpage(addr + i * PAGE_SIZE, i, arg);
   5611		}
   5612	} else {
   5613		/* If target subpage in second half of huge page */
   5614		base = pages_per_huge_page - 2 * (pages_per_huge_page - n);
   5615		l = pages_per_huge_page - n;
   5616		/* Process subpages at the begin of huge page */
   5617		for (i = 0; i < base; i++) {
   5618			cond_resched();
   5619			process_subpage(addr + i * PAGE_SIZE, i, arg);
   5620		}
   5621	}
   5622	/*
   5623	 * Process remaining subpages in left-right-left-right pattern
   5624	 * towards the target subpage
   5625	 */
   5626	for (i = 0; i < l; i++) {
   5627		int left_idx = base + i;
   5628		int right_idx = base + 2 * l - 1 - i;
   5629
   5630		cond_resched();
   5631		process_subpage(addr + left_idx * PAGE_SIZE, left_idx, arg);
   5632		cond_resched();
   5633		process_subpage(addr + right_idx * PAGE_SIZE, right_idx, arg);
   5634	}
   5635}
   5636
   5637static void clear_gigantic_page(struct page *page,
   5638				unsigned long addr,
   5639				unsigned int pages_per_huge_page)
   5640{
   5641	int i;
   5642	struct page *p = page;
   5643
   5644	might_sleep();
   5645	for (i = 0; i < pages_per_huge_page;
   5646	     i++, p = mem_map_next(p, page, i)) {
   5647		cond_resched();
   5648		clear_user_highpage(p, addr + i * PAGE_SIZE);
   5649	}
   5650}
   5651
   5652static void clear_subpage(unsigned long addr, int idx, void *arg)
   5653{
   5654	struct page *page = arg;
   5655
   5656	clear_user_highpage(page + idx, addr);
   5657}
   5658
   5659void clear_huge_page(struct page *page,
   5660		     unsigned long addr_hint, unsigned int pages_per_huge_page)
   5661{
   5662	unsigned long addr = addr_hint &
   5663		~(((unsigned long)pages_per_huge_page << PAGE_SHIFT) - 1);
   5664
   5665	if (unlikely(pages_per_huge_page > MAX_ORDER_NR_PAGES)) {
   5666		clear_gigantic_page(page, addr, pages_per_huge_page);
   5667		return;
   5668	}
   5669
   5670	process_huge_page(addr_hint, pages_per_huge_page, clear_subpage, page);
   5671}
   5672
   5673static void copy_user_gigantic_page(struct page *dst, struct page *src,
   5674				    unsigned long addr,
   5675				    struct vm_area_struct *vma,
   5676				    unsigned int pages_per_huge_page)
   5677{
   5678	int i;
   5679	struct page *dst_base = dst;
   5680	struct page *src_base = src;
   5681
   5682	for (i = 0; i < pages_per_huge_page; ) {
   5683		cond_resched();
   5684		copy_user_highpage(dst, src, addr + i*PAGE_SIZE, vma);
   5685
   5686		i++;
   5687		dst = mem_map_next(dst, dst_base, i);
   5688		src = mem_map_next(src, src_base, i);
   5689	}
   5690}
   5691
   5692struct copy_subpage_arg {
   5693	struct page *dst;
   5694	struct page *src;
   5695	struct vm_area_struct *vma;
   5696};
   5697
   5698static void copy_subpage(unsigned long addr, int idx, void *arg)
   5699{
   5700	struct copy_subpage_arg *copy_arg = arg;
   5701
   5702	copy_user_highpage(copy_arg->dst + idx, copy_arg->src + idx,
   5703			   addr, copy_arg->vma);
   5704}
   5705
   5706void copy_user_huge_page(struct page *dst, struct page *src,
   5707			 unsigned long addr_hint, struct vm_area_struct *vma,
   5708			 unsigned int pages_per_huge_page)
   5709{
   5710	unsigned long addr = addr_hint &
   5711		~(((unsigned long)pages_per_huge_page << PAGE_SHIFT) - 1);
   5712	struct copy_subpage_arg arg = {
   5713		.dst = dst,
   5714		.src = src,
   5715		.vma = vma,
   5716	};
   5717
   5718	if (unlikely(pages_per_huge_page > MAX_ORDER_NR_PAGES)) {
   5719		copy_user_gigantic_page(dst, src, addr, vma,
   5720					pages_per_huge_page);
   5721		return;
   5722	}
   5723
   5724	process_huge_page(addr_hint, pages_per_huge_page, copy_subpage, &arg);
   5725}
   5726
   5727long copy_huge_page_from_user(struct page *dst_page,
   5728				const void __user *usr_src,
   5729				unsigned int pages_per_huge_page,
   5730				bool allow_pagefault)
   5731{
   5732	void *page_kaddr;
   5733	unsigned long i, rc = 0;
   5734	unsigned long ret_val = pages_per_huge_page * PAGE_SIZE;
   5735	struct page *subpage = dst_page;
   5736
   5737	for (i = 0; i < pages_per_huge_page;
   5738	     i++, subpage = mem_map_next(subpage, dst_page, i)) {
   5739		if (allow_pagefault)
   5740			page_kaddr = kmap(subpage);
   5741		else
   5742			page_kaddr = kmap_atomic(subpage);
   5743		rc = copy_from_user(page_kaddr,
   5744				usr_src + i * PAGE_SIZE, PAGE_SIZE);
   5745		if (allow_pagefault)
   5746			kunmap(subpage);
   5747		else
   5748			kunmap_atomic(page_kaddr);
   5749
   5750		ret_val -= (PAGE_SIZE - rc);
   5751		if (rc)
   5752			break;
   5753
   5754		flush_dcache_page(subpage);
   5755
   5756		cond_resched();
   5757	}
   5758	return ret_val;
   5759}
   5760#endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_HUGETLBFS */
   5761
   5762#if USE_SPLIT_PTE_PTLOCKS && ALLOC_SPLIT_PTLOCKS
   5763
   5764static struct kmem_cache *page_ptl_cachep;
   5765
   5766void __init ptlock_cache_init(void)
   5767{
   5768	page_ptl_cachep = kmem_cache_create("page->ptl", sizeof(spinlock_t), 0,
   5769			SLAB_PANIC, NULL);
   5770}
   5771
   5772bool ptlock_alloc(struct page *page)
   5773{
   5774	spinlock_t *ptl;
   5775
   5776	ptl = kmem_cache_alloc(page_ptl_cachep, GFP_KERNEL);
   5777	if (!ptl)
   5778		return false;
   5779	page->ptl = ptl;
   5780	return true;
   5781}
   5782
   5783void ptlock_free(struct page *page)
   5784{
   5785	kmem_cache_free(page_ptl_cachep, page->ptl);
   5786}
   5787#endif