1 /* SPDX-License-Identifier: GPL-2.0 */
3  * Macros for manipulating and testing page->flags
18  * Various page->flags bits:
23  * - Pages part of the kernel image (including vDSO) and similar (e.g. BIOS,
25  * - Pages reserved or allocated early during boot (before the page allocator
30  * - Pages falling into physical memory gaps - not IORESOURCE_SYSRAM. Trying
32  * - The zero page(s)
33  * - Pages allocated in the context of kexec/kdump (loaded kernel image,
35  * - MMIO/DMA pages. Some architectures don't allow to ioremap pages that are
38  * - MCA pages on ia64
39  * - Pages holding CPU notes for POWER Firmware Assisted Dump
40  * - Device memory (e.g. PMEM, DAX, HMM)
45  * the zero page, the vDSO, MMIO pages or device memory.
48  * specific data (which is normally at page->private). It can be used by
67  * file-backed pagecache (see mm/vmscan.c).
86  *  N-1           ^       0
108 	PG_private,		/* If pagecache, has fs-private data */
141 	 * table (e.g., single PMD/PUD/CONT of the head page vs. PTE-mapped
144 	 * set on tail pages for PTE-mapped THP.
154 	/* Two page bits are conscripted by FS-Cache to maintain local caching
161 	/* Pinned in Xen as a read-only pagetable page. */
167 	/* Remapped by swiotlb-xen. */
170 	/* non-lru isolated movable page */
177 	/* For self-hosted memmap pages */
182 	 * Flags only valid for compound pages.  Stored in first tail page's
189 	PG_large_rmappable = PG_workingset, /* anon or file-backed */
193 #define PAGEFLAGS_MASK		((1UL << NR_PAGEFLAGS) - 1)
216 	    test_bit(PG_head, &page->flags)) {  in page_fixed_fake_head()
219 		 * because the @page is a compound page composed with at least  in page_fixed_fake_head()
225 			return (const struct page *)(head - 1);  in page_fixed_fake_head()
243 	unsigned long head = READ_ONCE(page->compound_head);  in _compound_head()
246 		return head - 1;  in _compound_head()
253  * page_folio - Converts from page to folio.
261  * it should re-check the folio still contains this page after gaining
270  * folio_page - Return a page from a folio.
278 #define folio_page(folio, n)	nth_page(&(folio)->page, n)
282 	return READ_ONCE(page->compound_head) & 1 || page_is_fake_head(page);  in PageTail()
287 	return test_bit(PG_head, &page->flags) ||  in PageCompound()
288 	       READ_ONCE(page->compound_head) & 1;  in PageCompound()
291 #define	PAGE_POISON_PATTERN	-1l
294 	return READ_ONCE(page->flags) == PAGE_POISON_PATTERN;  in PagePoisoned()
308 	const struct page *page = &folio->page;  in const_folio_flags()
310 	VM_BUG_ON_PGFLAGS(page->compound_head & 1, page);  in const_folio_flags()
311 	VM_BUG_ON_PGFLAGS(n > 0 && !test_bit(PG_head, &page->flags), page);  in const_folio_flags()
317 	struct page *page = &folio->page;  in folio_flags()
319 	VM_BUG_ON_PGFLAGS(page->compound_head & 1, page);  in folio_flags()
320 	VM_BUG_ON_PGFLAGS(n > 0 && !test_bit(PG_head, &page->flags), page);  in folio_flags()
325  * Page flags policies wrt compound pages
334  *     for compound page all operations related to the page flag applied to
342  *     the page flag is not relevant for compound pages.
411 { return test_bit(PG_##lname, &policy(page, 0)->flags); }
416 { set_bit(PG_##lname, &policy(page, 1)->flags); }
421 { clear_bit(PG_##lname, &policy(page, 1)->flags); }
426 { __set_bit(PG_##lname, &policy(page, 1)->flags); }
431 { __clear_bit(PG_##lname, &policy(page, 1)->flags); }
436 { return test_and_set_bit(PG_##lname, &policy(page, 1)->flags); }
441 { return test_and_clear_bit(PG_##lname, &policy(page, 1)->flags); }
544  * - PG_private and PG_private_2 cause release_folio() and co to be invoked  in PAGEFLAG()
553  * Only test-and-set exist for PG_writeback.  The unconditional operators are  in PAGEFLAG()
635  * allocator. We can use the non-atomic version of the test and set  in FOLIO_SET_FLAG()
649  * folio->mapping points to its anon_vma, not to a struct address_space;  in FOLIO_SET_FLAG()
654  * bit; and then folio->mapping points, not to an anon_vma, but to a private  in FOLIO_SET_FLAG()
657  * PAGE_MAPPING_KSM without PAGE_MAPPING_ANON is used for non-lru movable  in FOLIO_SET_FLAG()
658  * page and then folio->mapping points to a struct movable_operations.  in FOLIO_SET_FLAG()
665  * internal states, the folio->mapping does not exist as such, nor do  in FOLIO_SET_FLAG()
666  * these flags below.  So in order to avoid testing non-existent bits,  in FOLIO_SET_FLAG()
678  * indicates that this page->mapping is now under reflink case.  in FOLIO_SET_FLAG()
684 	return ((unsigned long)folio->mapping & PAGE_MAPPING_FLAGS) != 0;  in FOLIO_SET_FLAG()
689 	return ((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) != 0;  in PageMappingFlags()
694 	return ((unsigned long)folio->mapping & PAGE_MAPPING_ANON) != 0;  in folio_test_anon()
699 	unsigned long flags = (unsigned long)page_folio(page)->mapping;  in PageAnonNotKsm()
711 	return ((unsigned long)folio->mapping & PAGE_MAPPING_FLAGS) ==  in __folio_test_movable()
717 	return ((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) ==  in __PageMovable()
723  * A KSM page is one of those write-protected "shared pages" or "merged pages"
730 	return ((unsigned long)folio->mapping & PAGE_MAPPING_FLAGS) ==  in folio_test_ksm()
740  * folio_xor_flags_has_waiters - Change some folio flags.
747  * on flags which are in the range 0-6 as some of the implementations
759  * folio_test_uptodate - Is this folio up to date?
773 	 * _after_ we've loaded folio->flags to check the uptodate bit.  in folio_test_uptodate()
835 	return test_bit(PG_head, &page->flags) && !page_is_fake_head(page);  in PageHead()
843  * folio_test_large() - Does this folio contain more than one page?  in __SETPAGEFLAG()
855 	WRITE_ONCE(page->compound_head, (unsigned long)head + 1);  in set_compound_head()
860 	WRITE_ONCE(page->compound_head, 0);  in clear_compound_head()
910  * compound page.
926 	/* 0x00-0x7f are positive numbers, ie mapcount */
927 	/* Reserve 0x80-0xef for mapcount overflow. */
945 /* This takes a mapcount which is one more than page->_mapcount */
948 	return page_type_has_type(mapcount - 1);  in page_mapcount_is_type()
953 	return page_mapcount_is_type(data_race(page->page_type));  in page_has_type()
959 	return data_race(folio->page.page_type >> 24) == PGTY_##lname;	\
965 	VM_BUG_ON_FOLIO(data_race(folio->page.page_type) != UINT_MAX,	\
967 	folio->page.page_type = (unsigned int)PGTY_##lname << 24;	\
971 	if (folio->page.page_type == UINT_MAX)				\
974 	folio->page.page_type = UINT_MAX;				\
981 	return data_race(page->page_type >> 24) == PGTY_##lname;	\
987 	VM_BUG_ON_PAGE(data_race(page->page_type) != UINT_MAX, page);	\
988 	page->page_type = (unsigned int)PGTY_##lname << 24;		\
992 	if (page->page_type == UINT_MAX)				\
995 	page->page_type = UINT_MAX;					\
1021  * relies on this feature is aware that re-onlining the memory block will
1053  * PageSlab - Determine if the page belongs to the slab allocator  in PAGE_TYPE_OPS()
1080  * PageHuge - Determine if the page belongs to hugetlbfs  in FOLIO_TYPE_OPS()
1104 	return folio_test_hugetlb(folio) && PageHWPoison(&folio->page);  in is_page_hwpoison()
1126 	return test_bit(PG_anon_exclusive, &PF_ANY(page, 1)->flags);  in PageAnonExclusive()
1133 	set_bit(PG_anon_exclusive, &PF_ANY(page, 1)->flags);  in SetPageAnonExclusive()
1140 	clear_bit(PG_anon_exclusive, &PF_ANY(page, 1)->flags);  in ClearPageAnonExclusive()
1147 	__clear_bit(PG_anon_exclusive, &PF_ANY(page, 1)->flags);  in __ClearPageAnonExclusive()
1173  * alloc-free cycle to prevent from reusing the page.
1179  * Flags stored in the second page of a compound page.  They may overlap
1189  * folio_has_private - Determine if folio has private stuff
1197 	return !!(folio->flags & PAGE_FLAGS_PRIVATE);  in folio_has_private()