diff options
Diffstat (limited to 'mm')
| -rw-r--r-- | mm/balloon_compaction.c | 6 | ||||
| -rw-r--r-- | mm/damon/core.c | 5 | ||||
| -rw-r--r-- | mm/gup.c | 10 | ||||
| -rw-r--r-- | mm/internal.h | 6 | ||||
| -rw-r--r-- | mm/kfence/core.c | 11 | ||||
| -rw-r--r-- | mm/kfence/kfence.h | 3 | ||||
| -rw-r--r-- | mm/kmemleak.c | 9 | ||||
| -rw-r--r-- | mm/madvise.c | 9 | ||||
| -rw-r--r-- | mm/memory.c | 12 | ||||
| -rw-r--r-- | mm/migrate.c | 2 | ||||
| -rw-r--r-- | mm/mlock.c | 46 | ||||
| -rw-r--r-- | mm/page_alloc.c | 4 | ||||
| -rw-r--r-- | mm/rmap.c | 4 | ||||
| -rw-r--r-- | mm/swap.c | 4 |
14 files changed, 94 insertions, 37 deletions
diff --git a/mm/balloon_compaction.c b/mm/balloon_compaction.c index 907fefde2572..4b8eab4b3f45 100644 --- a/mm/balloon_compaction.c +++ b/mm/balloon_compaction.c @@ -203,7 +203,7 @@ EXPORT_SYMBOL_GPL(balloon_page_dequeue); #ifdef CONFIG_BALLOON_COMPACTION -bool balloon_page_isolate(struct page *page, isolate_mode_t mode) +static bool balloon_page_isolate(struct page *page, isolate_mode_t mode) { struct balloon_dev_info *b_dev_info = balloon_page_device(page); @@ -217,7 +217,7 @@ bool balloon_page_isolate(struct page *page, isolate_mode_t mode) return true; } -void balloon_page_putback(struct page *page) +static void balloon_page_putback(struct page *page) { struct balloon_dev_info *b_dev_info = balloon_page_device(page); unsigned long flags; @@ -230,7 +230,7 @@ void balloon_page_putback(struct page *page) /* move_to_new_page() counterpart for a ballooned page */ -int balloon_page_migrate(struct address_space *mapping, +static int balloon_page_migrate(struct address_space *mapping, struct page *newpage, struct page *page, enum migrate_mode mode) { diff --git a/mm/damon/core.c b/mm/damon/core.c index c1e0fed4e877..5ce8d7c867f0 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -1019,12 +1019,15 @@ static int kdamond_wait_activation(struct damon_ctx *ctx) struct damos *s; unsigned long wait_time; unsigned long min_wait_time = 0; + bool init_wait_time = false; while (!kdamond_need_stop(ctx)) { damon_for_each_scheme(s, ctx) { wait_time = damos_wmark_wait_us(s); - if (!min_wait_time || wait_time < min_wait_time) + if (!init_wait_time || wait_time < min_wait_time) { + init_wait_time = true; min_wait_time = wait_time; + } } if (!min_wait_time) return 0; @@ -1404,6 +1404,7 @@ long populate_vma_page_range(struct vm_area_struct *vma, struct mm_struct *mm = vma->vm_mm; unsigned long nr_pages = (end - start) / PAGE_SIZE; int gup_flags; + long ret; VM_BUG_ON(!PAGE_ALIGNED(start)); VM_BUG_ON(!PAGE_ALIGNED(end)); @@ -1438,8 +1439,10 @@ long populate_vma_page_range(struct vm_area_struct *vma, * We made sure addr is within a VMA, so the following will * not result in a stack expansion that recurses back here. */ - return __get_user_pages(mm, start, nr_pages, gup_flags, + ret = __get_user_pages(mm, start, nr_pages, gup_flags, NULL, NULL, locked); + lru_add_drain(); + return ret; } /* @@ -1471,6 +1474,7 @@ long faultin_vma_page_range(struct vm_area_struct *vma, unsigned long start, struct mm_struct *mm = vma->vm_mm; unsigned long nr_pages = (end - start) / PAGE_SIZE; int gup_flags; + long ret; VM_BUG_ON(!PAGE_ALIGNED(start)); VM_BUG_ON(!PAGE_ALIGNED(end)); @@ -1498,8 +1502,10 @@ long faultin_vma_page_range(struct vm_area_struct *vma, unsigned long start, if (check_vma_flags(vma, gup_flags)) return -EINVAL; - return __get_user_pages(mm, start, nr_pages, gup_flags, + ret = __get_user_pages(mm, start, nr_pages, gup_flags, NULL, NULL, locked); + lru_add_drain(); + return ret; } /* diff --git a/mm/internal.h b/mm/internal.h index 58dc6adc19c5..cf16280ce132 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -456,7 +456,8 @@ static inline void munlock_vma_page(struct page *page, } void mlock_new_page(struct page *page); bool need_mlock_page_drain(int cpu); -void mlock_page_drain(int cpu); +void mlock_page_drain_local(void); +void mlock_page_drain_remote(int cpu); extern pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma); @@ -539,7 +540,8 @@ static inline void munlock_vma_page(struct page *page, struct vm_area_struct *vma, bool compound) { } static inline void mlock_new_page(struct page *page) { } static inline bool need_mlock_page_drain(int cpu) { return false; } -static inline void mlock_page_drain(int cpu) { } +static inline void mlock_page_drain_local(void) { } +static inline void mlock_page_drain_remote(int cpu) { } static inline void vunmap_range_noflush(unsigned long start, unsigned long end) { } diff --git a/mm/kfence/core.c b/mm/kfence/core.c index 2f9fdfde1941..a203747ad2c0 100644 --- a/mm/kfence/core.c +++ b/mm/kfence/core.c @@ -566,6 +566,8 @@ static unsigned long kfence_init_pool(void) * enters __slab_free() slow-path. */ for (i = 0; i < KFENCE_POOL_SIZE / PAGE_SIZE; i++) { + struct slab *slab = page_slab(&pages[i]); + if (!i || (i % 2)) continue; @@ -573,7 +575,11 @@ static unsigned long kfence_init_pool(void) if (WARN_ON(compound_head(&pages[i]) != &pages[i])) return addr; - __SetPageSlab(&pages[i]); + __folio_set_slab(slab_folio(slab)); +#ifdef CONFIG_MEMCG + slab->memcg_data = (unsigned long)&kfence_metadata[i / 2 - 1].objcg | + MEMCG_DATA_OBJCGS; +#endif } /* @@ -1033,6 +1039,9 @@ void __kfence_free(void *addr) { struct kfence_metadata *meta = addr_to_metadata((unsigned long)addr); +#ifdef CONFIG_MEMCG + KFENCE_WARN_ON(meta->objcg); +#endif /* * If the objects of the cache are SLAB_TYPESAFE_BY_RCU, defer freeing * the object, as the object page may be recycled for other-typed diff --git a/mm/kfence/kfence.h b/mm/kfence/kfence.h index 2a2d5de9d379..9a6c4b1b12a8 100644 --- a/mm/kfence/kfence.h +++ b/mm/kfence/kfence.h @@ -89,6 +89,9 @@ struct kfence_metadata { struct kfence_track free_track; /* For updating alloc_covered on frees. */ u32 alloc_stack_hash; +#ifdef CONFIG_MEMCG + struct obj_cgroup *objcg; +#endif }; extern struct kfence_metadata kfence_metadata[CONFIG_KFENCE_NUM_OBJECTS]; diff --git a/mm/kmemleak.c b/mm/kmemleak.c index 7580baa76af1..acd7cbb82e16 100644 --- a/mm/kmemleak.c +++ b/mm/kmemleak.c @@ -796,6 +796,8 @@ static void add_scan_area(unsigned long ptr, size_t size, gfp_t gfp) unsigned long flags; struct kmemleak_object *object; struct kmemleak_scan_area *area = NULL; + unsigned long untagged_ptr; + unsigned long untagged_objp; object = find_and_get_object(ptr, 1); if (!object) { @@ -804,6 +806,9 @@ static void add_scan_area(unsigned long ptr, size_t size, gfp_t gfp) return; } + untagged_ptr = (unsigned long)kasan_reset_tag((void *)ptr); + untagged_objp = (unsigned long)kasan_reset_tag((void *)object->pointer); + if (scan_area_cache) area = kmem_cache_alloc(scan_area_cache, gfp_kmemleak_mask(gfp)); @@ -815,8 +820,8 @@ static void add_scan_area(unsigned long ptr, size_t size, gfp_t gfp) goto out_unlock; } if (size == SIZE_MAX) { - size = object->pointer + object->size - ptr; - } else if (ptr + size > object->pointer + object->size) { + size = untagged_objp + object->size - untagged_ptr; + } else if (untagged_ptr + size > untagged_objp + object->size) { kmemleak_warn("Scan area larger than object 0x%08lx\n", ptr); dump_object_info(object); kmem_cache_free(scan_area_cache, area); diff --git a/mm/madvise.c b/mm/madvise.c index b41858ee937b..1873616a37d2 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -1464,16 +1464,9 @@ SYSCALL_DEFINE5(process_madvise, int, pidfd, const struct iovec __user *, vec, while (iov_iter_count(&iter)) { iovec = iov_iter_iovec(&iter); - /* - * do_madvise returns ENOMEM if unmapped holes are present - * in the passed VMA. process_madvise() is expected to skip - * unmapped holes passed to it in the 'struct iovec' list - * and not fail because of them. Thus treat -ENOMEM return - * from do_madvise as valid and continue processing. - */ ret = do_madvise(mm, (unsigned long)iovec.iov_base, iovec.iov_len, behavior); - if (ret < 0 && ret != -ENOMEM) + if (ret < 0) break; iov_iter_advance(&iter, iovec.iov_len); } diff --git a/mm/memory.c b/mm/memory.c index be44d0b36b18..76e3af9639d9 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -3918,14 +3918,18 @@ static vm_fault_t __do_fault(struct vm_fault *vmf) return ret; if (unlikely(PageHWPoison(vmf->page))) { + struct page *page = vmf->page; vm_fault_t poisonret = VM_FAULT_HWPOISON; if (ret & VM_FAULT_LOCKED) { + if (page_mapped(page)) + unmap_mapping_pages(page_mapping(page), + page->index, 1, false); /* Retry if a clean page was removed from the cache. */ - if (invalidate_inode_page(vmf->page)) - poisonret = 0; - unlock_page(vmf->page); + if (invalidate_inode_page(page)) + poisonret = VM_FAULT_NOPAGE; + unlock_page(page); } - put_page(vmf->page); + put_page(page); vmf->page = NULL; return poisonret; } diff --git a/mm/migrate.c b/mm/migrate.c index 3d60823afd2d..de175e2fdba5 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -246,7 +246,7 @@ static bool remove_migration_pte(struct folio *folio, set_pte_at(vma->vm_mm, pvmw.address, pvmw.pte, pte); } if (vma->vm_flags & VM_LOCKED) - mlock_page_drain(smp_processor_id()); + mlock_page_drain_local(); trace_remove_migration_pte(pvmw.address, pte_val(pte), compound_order(new)); diff --git a/mm/mlock.c b/mm/mlock.c index 529fbc1f27c8..716caf851043 100644 --- a/mm/mlock.c +++ b/mm/mlock.c @@ -28,7 +28,14 @@ #include "internal.h" -static DEFINE_PER_CPU(struct pagevec, mlock_pvec); +struct mlock_pvec { + local_lock_t lock; + struct pagevec vec; +}; + +static DEFINE_PER_CPU(struct mlock_pvec, mlock_pvec) = { + .lock = INIT_LOCAL_LOCK(lock), +}; bool can_do_mlock(void) { @@ -203,18 +210,30 @@ static void mlock_pagevec(struct pagevec *pvec) pagevec_reinit(pvec); } -void mlock_page_drain(int cpu) +void mlock_page_drain_local(void) +{ + struct pagevec *pvec; + + local_lock(&mlock_pvec.lock); + pvec = this_cpu_ptr(&mlock_pvec.vec); + if (pagevec_count(pvec)) + mlock_pagevec(pvec); + local_unlock(&mlock_pvec.lock); +} + +void mlock_page_drain_remote(int cpu) { struct pagevec *pvec; - pvec = &per_cpu(mlock_pvec, cpu); + WARN_ON_ONCE(cpu_online(cpu)); + pvec = &per_cpu(mlock_pvec.vec, cpu); if (pagevec_count(pvec)) mlock_pagevec(pvec); } bool need_mlock_page_drain(int cpu) { - return pagevec_count(&per_cpu(mlock_pvec, cpu)); + return pagevec_count(&per_cpu(mlock_pvec.vec, cpu)); } /** @@ -223,7 +242,10 @@ bool need_mlock_page_drain(int cpu) */ void mlock_folio(struct folio *folio) { - struct pagevec *pvec = &get_cpu_var(mlock_pvec); + struct pagevec *pvec; + + local_lock(&mlock_pvec.lock); + pvec = this_cpu_ptr(&mlock_pvec.vec); if (!folio_test_set_mlocked(folio)) { int nr_pages = folio_nr_pages(folio); @@ -236,7 +258,7 @@ void mlock_folio(struct folio *folio) if (!pagevec_add(pvec, mlock_lru(&folio->page)) || folio_test_large(folio) || lru_cache_disabled()) mlock_pagevec(pvec); - put_cpu_var(mlock_pvec); + local_unlock(&mlock_pvec.lock); } /** @@ -245,9 +267,11 @@ void mlock_folio(struct folio *folio) */ void mlock_new_page(struct page *page) { - struct pagevec *pvec = &get_cpu_var(mlock_pvec); + struct pagevec *pvec; int nr_pages = thp_nr_pages(page); + local_lock(&mlock_pvec.lock); + pvec = this_cpu_ptr(&mlock_pvec.vec); SetPageMlocked(page); mod_zone_page_state(page_zone(page), NR_MLOCK, nr_pages); __count_vm_events(UNEVICTABLE_PGMLOCKED, nr_pages); @@ -256,7 +280,7 @@ void mlock_new_page(struct page *page) if (!pagevec_add(pvec, mlock_new(page)) || PageHead(page) || lru_cache_disabled()) mlock_pagevec(pvec); - put_cpu_var(mlock_pvec); + local_unlock(&mlock_pvec.lock); } /** @@ -265,8 +289,10 @@ void mlock_new_page(struct page *page) */ void munlock_page(struct page *page) { - struct pagevec *pvec = &get_cpu_var(mlock_pvec); + struct pagevec *pvec; + local_lock(&mlock_pvec.lock); + pvec = this_cpu_ptr(&mlock_pvec.vec); /* * TestClearPageMlocked(page) must be left to __munlock_page(), * which will check whether the page is multiply mlocked. @@ -276,7 +302,7 @@ void munlock_page(struct page *page) if (!pagevec_add(pvec, page) || PageHead(page) || lru_cache_disabled()) mlock_pagevec(pvec); - put_cpu_var(mlock_pvec); + local_unlock(&mlock_pvec.lock); } static int mlock_pte_range(pmd_t *pmd, unsigned long addr, diff --git a/mm/page_alloc.c b/mm/page_alloc.c index bdc8f60ae462..2db95780e003 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -1108,6 +1108,9 @@ continue_merging: buddy_pfn = __find_buddy_pfn(pfn, order); buddy = page + (buddy_pfn - pfn); + + if (!page_is_buddy(page, buddy, order)) + goto done_merging; buddy_mt = get_pageblock_migratetype(buddy); if (migratetype != buddy_mt @@ -8364,6 +8367,7 @@ static int page_alloc_cpu_dead(unsigned int cpu) struct zone *zone; lru_add_drain_cpu(cpu); + mlock_page_drain_remote(cpu); drain_pages(cpu); /* diff --git a/mm/rmap.c b/mm/rmap.c index 5cb970d51f0a..fedb82371efe 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -1683,7 +1683,7 @@ discard: */ page_remove_rmap(subpage, vma, folio_test_hugetlb(folio)); if (vma->vm_flags & VM_LOCKED) - mlock_page_drain(smp_processor_id()); + mlock_page_drain_local(); folio_put(folio); } @@ -1961,7 +1961,7 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, */ page_remove_rmap(subpage, vma, folio_test_hugetlb(folio)); if (vma->vm_flags & VM_LOCKED) - mlock_page_drain(smp_processor_id()); + mlock_page_drain_local(); folio_put(folio); } diff --git a/mm/swap.c b/mm/swap.c index bceff0cb559c..7e320ec08c6a 100644 --- a/mm/swap.c +++ b/mm/swap.c @@ -624,7 +624,6 @@ void lru_add_drain_cpu(int cpu) pagevec_lru_move_fn(pvec, lru_lazyfree_fn); activate_page_drain(cpu); - mlock_page_drain(cpu); } /** @@ -706,6 +705,7 @@ void lru_add_drain(void) local_lock(&lru_pvecs.lock); lru_add_drain_cpu(smp_processor_id()); local_unlock(&lru_pvecs.lock); + mlock_page_drain_local(); } /* @@ -720,6 +720,7 @@ static void lru_add_and_bh_lrus_drain(void) lru_add_drain_cpu(smp_processor_id()); local_unlock(&lru_pvecs.lock); invalidate_bh_lrus_cpu(); + mlock_page_drain_local(); } void lru_add_drain_cpu_zone(struct zone *zone) @@ -728,6 +729,7 @@ void lru_add_drain_cpu_zone(struct zone *zone) lru_add_drain_cpu(smp_processor_id()); drain_local_pages(zone); local_unlock(&lru_pvecs.lock); + mlock_page_drain_local(); } #ifdef CONFIG_SMP |