150 static void anon_vma_chain_link(struct vm_area_struct *vma,  in anon_vma_chain_link()  argument
154 	avc->vma = vma;  in anon_vma_chain_link()
156 	list_add(&avc->same_vma, &vma->anon_vma_chain);  in anon_vma_chain_link()
186 int __anon_vma_prepare(struct vm_area_struct *vma)  in __anon_vma_prepare()  argument
188 	struct mm_struct *mm = vma->vm_mm;  in __anon_vma_prepare()
199 	anon_vma = find_mergeable_anon_vma(vma);  in __anon_vma_prepare()
212 	if (likely(!vma->anon_vma)) {  in __anon_vma_prepare()
213 		vma->anon_vma = anon_vma;  in __anon_vma_prepare()
214 		anon_vma_chain_link(vma, avc, anon_vma);  in __anon_vma_prepare()
334 int anon_vma_fork(struct vm_area_struct *vma, struct vm_area_struct *pvma)  in anon_vma_fork()  argument
345 	vma->anon_vma = NULL;  in anon_vma_fork()
351 	error = anon_vma_clone(vma, pvma);  in anon_vma_fork()
356 	if (vma->anon_vma)  in anon_vma_fork()
381 	vma->anon_vma = anon_vma;  in anon_vma_fork()
383 	anon_vma_chain_link(vma, avc, anon_vma);  in anon_vma_fork()
392 	unlink_anon_vmas(vma);  in anon_vma_fork()
396 void unlink_anon_vmas(struct vm_area_struct *vma)  in unlink_anon_vmas()  argument
405 	list_for_each_entry_safe(avc, next, &vma->anon_vma_chain, same_vma) {  in unlink_anon_vmas()
423 	if (vma->anon_vma) {  in unlink_anon_vmas()
424 		vma->anon_vma->num_active_vmas--;  in unlink_anon_vmas()
430 		vma->anon_vma = NULL;  in unlink_anon_vmas()
439 	list_for_each_entry_safe(avc, next, &vma->anon_vma_chain, same_vma) {  in unlink_anon_vmas()
789 		const struct page *page, const struct vm_area_struct *vma)  in page_address_in_vma()  argument
797 		if (!vma->anon_vma || !page__anon_vma ||  in page_address_in_vma()
798 		    vma->anon_vma->root != page__anon_vma->root)  in page_address_in_vma()
800 	} else if (!vma->vm_file) {  in page_address_in_vma()
802 	} else if (vma->vm_file->f_mapping != folio->mapping) {  in page_address_in_vma()
807 	return vma_address(vma, page_pgoff(folio, page), 1);  in page_address_in_vma()
850 		struct vm_area_struct *vma, unsigned long address, void *arg)  in folio_referenced_one()  argument
853 	DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0);  in folio_referenced_one()
860 		if (vma->vm_flags & VM_LOCKED) {  in folio_referenced_one()
863 				mlock_vma_folio(folio, vma);  in folio_referenced_one()
889 		if ((!atomic_read(&vma->vm_mm->mm_users) ||  in folio_referenced_one()
890 		    check_stable_address_space(vma->vm_mm)) &&  in folio_referenced_one()
902 			if (ptep_clear_flush_young_notify(vma, address,  in folio_referenced_one()
906 			if (pmdp_clear_flush_young_notify(vma, address,  in folio_referenced_one()
917 	if ((vma->vm_flags & VM_LOCKED) &&  in folio_referenced_one()
919 			folio_within_vma(folio, vma)) {  in folio_referenced_one()
928 			mlock_vma_folio(folio, vma);  in folio_referenced_one()
941 		pra->vm_flags |= vma->vm_flags & ~VM_LOCKED;  in folio_referenced_one()
950 static bool invalid_folio_referenced_vma(struct vm_area_struct *vma, void *arg)  in invalid_folio_referenced_vma()  argument
961 	if (!vma_has_recency(vma))  in invalid_folio_referenced_vma()
968 	if (memcg && !mm_match_cgroup(vma->vm_mm, memcg))  in invalid_folio_referenced_vma()
1027 	struct vm_area_struct *vma = pvmw->vma;  in page_vma_mkclean_one()  local
1036 				vma->vm_mm, address, vma_address_end(pvmw));  in page_vma_mkclean_one()
1050 			flush_cache_page(vma, address, pte_pfn(entry));  in page_vma_mkclean_one()
1051 			entry = ptep_clear_flush(vma, address, pte);  in page_vma_mkclean_one()
1054 			set_pte_at(vma->vm_mm, address, pte, entry);  in page_vma_mkclean_one()
1064 			flush_cache_range(vma, address,  in page_vma_mkclean_one()
1066 			entry = pmdp_invalidate(vma, address, pmd);  in page_vma_mkclean_one()
1069 			set_pmd_at(vma->vm_mm, address, pmd, entry);  in page_vma_mkclean_one()
1086 static bool page_mkclean_one(struct folio *folio, struct vm_area_struct *vma,  in page_mkclean_one()  argument
1089 	DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, PVMW_SYNC);  in page_mkclean_one()
1097 static bool invalid_mkclean_vma(struct vm_area_struct *vma, void *arg)  in invalid_mkclean_vma()  argument
1099 	if (vma->vm_flags & VM_SHARED)  in invalid_mkclean_vma()
1143 		      struct vm_area_struct *vma)  in pfn_mkclean_range()  argument
1149 		.vma		= vma,  in pfn_mkclean_range()
1153 	if (invalid_mkclean_vma(vma, NULL))  in pfn_mkclean_range()
1156 	pvmw.address = vma_address(vma, pgoff, nr_pages);  in pfn_mkclean_range()
1157 	VM_BUG_ON_VMA(pvmw.address == -EFAULT, vma);  in pfn_mkclean_range()
1219 void folio_move_anon_rmap(struct folio *folio, struct vm_area_struct *vma)  in folio_move_anon_rmap()  argument
1221 	void *anon_vma = vma->anon_vma;  in folio_move_anon_rmap()
1224 	VM_BUG_ON_VMA(!anon_vma, vma);  in folio_move_anon_rmap()
1242 static void __folio_set_anon(struct folio *folio, struct vm_area_struct *vma,  in __folio_set_anon()  argument
1245 	struct anon_vma *anon_vma = vma->anon_vma;  in __folio_set_anon()
1264 	folio->index = linear_page_index(vma, address);  in __folio_set_anon()
1275 		const struct page *page, struct vm_area_struct *vma,  in __page_check_anon_rmap()  argument
1289 	VM_BUG_ON_FOLIO(folio_anon_vma(folio)->root != vma->anon_vma->root,  in __page_check_anon_rmap()
1291 	VM_BUG_ON_PAGE(page_pgoff(folio, page) != linear_page_index(vma, address),  in __page_check_anon_rmap()
1318 		struct page *page, int nr_pages, struct vm_area_struct *vma,  in __folio_add_anon_rmap()  argument
1328 		__page_check_anon_rmap(folio, page, vma, address);  in __folio_add_anon_rmap()
1360 		mlock_vma_folio(folio, vma);  in __folio_add_anon_rmap()
1380 		int nr_pages, struct vm_area_struct *vma, unsigned long address,  in folio_add_anon_rmap_ptes()  argument
1383 	__folio_add_anon_rmap(folio, page, nr_pages, vma, address, flags,  in folio_add_anon_rmap_ptes()
1401 		struct vm_area_struct *vma, unsigned long address, rmap_t flags)  in folio_add_anon_rmap_pmd()  argument
1404 	__folio_add_anon_rmap(folio, page, HPAGE_PMD_NR, vma, address, flags,  in folio_add_anon_rmap_pmd()
1426 void folio_add_new_anon_rmap(struct folio *folio, struct vm_area_struct *vma,  in folio_add_new_anon_rmap()  argument
1435 	VM_BUG_ON_VMA(address < vma->vm_start ||  in folio_add_new_anon_rmap()
1436 			address + (nr << PAGE_SHIFT) > vma->vm_end, vma);  in folio_add_new_anon_rmap()
1442 	if (!folio_test_swapbacked(folio) && !(vma->vm_flags & VM_DROPPABLE))  in folio_add_new_anon_rmap()
1444 	__folio_set_anon(folio, vma, address, exclusive);  in folio_add_new_anon_rmap()
1482 		struct page *page, int nr_pages, struct vm_area_struct *vma,  in __folio_add_file_rmap()  argument
1494 		mlock_vma_folio(folio, vma);  in __folio_add_file_rmap()
1509 		int nr_pages, struct vm_area_struct *vma)  in folio_add_file_rmap_ptes()  argument
1511 	__folio_add_file_rmap(folio, page, nr_pages, vma, RMAP_LEVEL_PTE);  in folio_add_file_rmap_ptes()
1525 		struct vm_area_struct *vma)  in folio_add_file_rmap_pmd()  argument
1528 	__folio_add_file_rmap(folio, page, HPAGE_PMD_NR, vma, RMAP_LEVEL_PMD);  in folio_add_file_rmap_pmd()
1535 		struct page *page, int nr_pages, struct vm_area_struct *vma,  in __folio_remove_rmap()  argument
1603 	munlock_vma_folio(folio, vma);  in __folio_remove_rmap()
1618 		int nr_pages, struct vm_area_struct *vma)  in folio_remove_rmap_ptes()  argument
1620 	__folio_remove_rmap(folio, page, nr_pages, vma, RMAP_LEVEL_PTE);  in folio_remove_rmap_ptes()
1634 		struct vm_area_struct *vma)  in folio_remove_rmap_pmd()  argument
1637 	__folio_remove_rmap(folio, page, HPAGE_PMD_NR, vma, RMAP_LEVEL_PMD);  in folio_remove_rmap_pmd()
1646 static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma,  in try_to_unmap_one()  argument
1649 	struct mm_struct *mm = vma->vm_mm;  in try_to_unmap_one()
1650 	DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0);  in try_to_unmap_one()
1677 	mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm,  in try_to_unmap_one()
1684 		adjust_range_if_pmd_sharing_possible(vma, &range.start,  in try_to_unmap_one()
1688 		hsz = huge_page_size(hstate_vma(vma));  in try_to_unmap_one()
1697 		    (vma->vm_flags & VM_LOCKED)) {  in try_to_unmap_one()
1700 				mlock_vma_folio(folio, vma);  in try_to_unmap_one()
1705 			if (unmap_huge_pmd_locked(vma, pvmw.address, pvmw.pmd,  in try_to_unmap_one()
1714 				split_huge_pmd_locked(vma, pvmw.address,  in try_to_unmap_one()
1746 			flush_cache_range(vma, range.start, range.end);  in try_to_unmap_one()
1760 				if (!hugetlb_vma_trylock_write(vma))  in try_to_unmap_one()
1762 				if (huge_pmd_unshare(mm, vma, address, pvmw.pte)) {  in try_to_unmap_one()
1763 					hugetlb_vma_unlock_write(vma);  in try_to_unmap_one()
1764 					flush_tlb_range(vma,  in try_to_unmap_one()
1778 				hugetlb_vma_unlock_write(vma);  in try_to_unmap_one()
1780 			pteval = huge_ptep_clear_flush(vma, address, pvmw.pte);  in try_to_unmap_one()
1782 			flush_cache_page(vma, address, pfn);  in try_to_unmap_one()
1797 				pteval = ptep_clear_flush(vma, address, pvmw.pte);  in try_to_unmap_one()
1806 		pte_install_uffd_wp_if_needed(vma, address, pvmw.pte, pteval);  in try_to_unmap_one()
1826 		} else if (pte_unused(pteval) && !userfaultfd_armed(vma)) {  in try_to_unmap_one()
1882 				     (vma->vm_flags & VM_DROPPABLE))) {  in try_to_unmap_one()
1896 				if (!(vma->vm_flags & VM_DROPPABLE))  in try_to_unmap_one()
1905 			if (arch_unmap_one(mm, vma, address, pteval) < 0) {  in try_to_unmap_one()
1952 			folio_remove_rmap_pte(folio, subpage, vma);  in try_to_unmap_one()
1953 		if (vma->vm_flags & VM_LOCKED)  in try_to_unmap_one()
1969 static bool invalid_migration_vma(struct vm_area_struct *vma, void *arg)  in invalid_migration_vma()  argument
1971 	return vma_is_temporary_stack(vma);  in invalid_migration_vma()
2011 static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma,  in try_to_migrate_one()  argument
2014 	struct mm_struct *mm = vma->vm_mm;  in try_to_migrate_one()
2015 	DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0);  in try_to_migrate_one()
2038 		split_huge_pmd_address(vma, address, true, folio);  in try_to_migrate_one()
2049 	mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm,  in try_to_migrate_one()
2056 		adjust_range_if_pmd_sharing_possible(vma, &range.start,  in try_to_migrate_one()
2060 		hsz = huge_page_size(hstate_vma(vma));  in try_to_migrate_one()
2117 			flush_cache_range(vma, range.start, range.end);  in try_to_migrate_one()
2131 				if (!hugetlb_vma_trylock_write(vma)) {  in try_to_migrate_one()
2136 				if (huge_pmd_unshare(mm, vma, address, pvmw.pte)) {  in try_to_migrate_one()
2137 					hugetlb_vma_unlock_write(vma);  in try_to_migrate_one()
2138 					flush_tlb_range(vma,  in try_to_migrate_one()
2154 				hugetlb_vma_unlock_write(vma);  in try_to_migrate_one()
2157 			pteval = huge_ptep_clear_flush(vma, address, pvmw.pte);  in try_to_migrate_one()
2159 			flush_cache_page(vma, address, pfn);  in try_to_migrate_one()
2174 				pteval = ptep_clear_flush(vma, address, pvmw.pte);  in try_to_migrate_one()
2234 		} else if (pte_unused(pteval) && !userfaultfd_armed(vma)) {  in try_to_migrate_one()
2250 			if (arch_unmap_one(mm, vma, address, pteval) < 0) {  in try_to_migrate_one()
2320 			folio_remove_rmap_pte(folio, subpage, vma);  in try_to_migrate_one()
2321 		if (vma->vm_flags & VM_LOCKED)  in try_to_migrate_one()
2386 		struct vm_area_struct *vma, unsigned long address, void *priv)  in page_make_device_exclusive_one()  argument
2388 	struct mm_struct *mm = vma->vm_mm;  in page_make_device_exclusive_one()
2389 	DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0);  in page_make_device_exclusive_one()
2400 				      vma->vm_mm, address, min(vma->vm_end,  in page_make_device_exclusive_one()
2421 		flush_cache_page(vma, address, pte_pfn(ptent));  in page_make_device_exclusive_one()
2422 		pteval = ptep_clear_flush(vma, address, pvmw.pte);  in page_make_device_exclusive_one()
2459 		folio_remove_rmap_pte(folio, subpage, vma);  in page_make_device_exclusive_one()
2636 		struct vm_area_struct *vma = avc->vma;  in rmap_walk_anon()  local
2637 		unsigned long address = vma_address(vma, pgoff_start,  in rmap_walk_anon()
2640 		VM_BUG_ON_VMA(address == -EFAULT, vma);  in rmap_walk_anon()
2643 		if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg))  in rmap_walk_anon()
2646 		if (!rwc->rmap_one(folio, vma, address, rwc->arg))  in rmap_walk_anon()
2670 	struct vm_area_struct *vma;  in rmap_walk_file()  local
2697 	vma_interval_tree_foreach(vma, &mapping->i_mmap,  in rmap_walk_file()
2699 		unsigned long address = vma_address(vma, pgoff_start,  in rmap_walk_file()
2702 		VM_BUG_ON_VMA(address == -EFAULT, vma);  in rmap_walk_file()
2705 		if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg))  in rmap_walk_file()
2708 		if (!rwc->rmap_one(folio, vma, address, rwc->arg))  in rmap_walk_file()
2746 void hugetlb_add_anon_rmap(struct folio *folio, struct vm_area_struct *vma,  in hugetlb_add_anon_rmap()  argument
2761 		struct vm_area_struct *vma, unsigned long address)  in hugetlb_add_new_anon_rmap()  argument
2765 	BUG_ON(address < vma->vm_start || address >= vma->vm_end);  in hugetlb_add_new_anon_rmap()
2770 	__folio_set_anon(folio, vma, address, true);  in hugetlb_add_new_anon_rmap()