163 	char *mem;  in gethugetlb_mem()  local
170 		mem = shmat(*shmid, 0, 0);  in gethugetlb_mem()
171 		if (mem == (char *)-1) {  in gethugetlb_mem()
176 		mem = mmap(NULL, size, PROT_READ | PROT_WRITE,  in gethugetlb_mem()
178 		if (mem == MAP_FAILED)  in gethugetlb_mem()
182 	return mem;  in gethugetlb_mem()
188 	char *mem, *vec;  in userfaultfd_tests()  local
191 	mem = mmap(NULL, mem_size, PROT_NONE, MAP_PRIVATE | MAP_ANON, -1, 0);  in userfaultfd_tests()
192 	if (mem == MAP_FAILED)  in userfaultfd_tests()
195 	wp_init(mem, mem_size);  in userfaultfd_tests()
198 	mprotect(mem, mem_size/8, PROT_READ|PROT_WRITE);  in userfaultfd_tests()
199 	mprotect(mem + 1 * mem_size/8, mem_size/8, PROT_READ);  in userfaultfd_tests()
200 	mprotect(mem + 2 * mem_size/8, mem_size/8, PROT_READ|PROT_WRITE);  in userfaultfd_tests()
201 	mprotect(mem + 3 * mem_size/8, mem_size/8, PROT_READ);  in userfaultfd_tests()
202 	mprotect(mem + 4 * mem_size/8, mem_size/8, PROT_READ|PROT_WRITE);  in userfaultfd_tests()
203 	mprotect(mem + 5 * mem_size/8, mem_size/8, PROT_NONE);  in userfaultfd_tests()
204 	mprotect(mem + 6 * mem_size/8, mem_size/8, PROT_READ|PROT_WRITE);  in userfaultfd_tests()
205 	mprotect(mem + 7 * mem_size/8, mem_size/8, PROT_READ);  in userfaultfd_tests()
207 	wp_addr_range(mem + (mem_size/16), mem_size - 2 * (mem_size/8));  in userfaultfd_tests()
208 	wp_addr_range(mem, mem_size);  in userfaultfd_tests()
213 	written = pagemap_ioctl(mem, mem_size, vec, 1, PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC,  in userfaultfd_tests()
220 	wp_free(mem, mem_size);  in userfaultfd_tests()
221 	munmap(mem, mem_size);  in userfaultfd_tests()
243 	char *mem, *m[2];  in sanity_tests_sd()  local
257 	mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0);  in sanity_tests_sd()
258 	if (mem == MAP_FAILED)  in sanity_tests_sd()
261 	wp_init(mem, mem_size);  in sanity_tests_sd()
262 	wp_addr_range(mem, mem_size);  in sanity_tests_sd()
265 	ksft_test_result(pagemap_ioctl(mem, 0, vec, vec_size, 0,  in sanity_tests_sd()
269 	ksft_test_result(pagemap_ioctl(mem, mem_size, NULL, vec_size, 0,  in sanity_tests_sd()
273 	ksft_test_result(pagemap_ioctl(mem, mem_size, vec, 0, 0,  in sanity_tests_sd()
277 	ksft_test_result(pagemap_ioctl(mem, mem_size, 0, 0, 0,  in sanity_tests_sd()
281 	ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, -1,  in sanity_tests_sd()
285 	ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size,  in sanity_tests_sd()
290 	ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, 0,  in sanity_tests_sd()
294 	ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, 0,  in sanity_tests_sd()
298 	ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, 0,  in sanity_tests_sd()
302 	ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size,  in sanity_tests_sd()
307 	ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size,  in sanity_tests_sd()
314 	ret = pagemap_ioctl(mem, mem_size, vec, vec_size,  in sanity_tests_sd()
321 		mem[i]++;  in sanity_tests_sd()
323 	ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0, PAGE_IS_WRITTEN, 0,  in sanity_tests_sd()
332 	ret = pagemap_ioctl(mem, mem_size, vec, vec_size,  in sanity_tests_sd()
338 	ret2 = pagemap_ioctl(mem, mem_size, vec, 2, 0, 0, PAGE_IS_WRITTEN, 0, 0,  in sanity_tests_sd()
343 	ret3 = pagemap_ioctl(mem, mem_size, vec, vec_size,  in sanity_tests_sd()
355 		mem[i]++;  in sanity_tests_sd()
356 	mem[(mem_size/page_size - 1) * page_size]++;  in sanity_tests_sd()
358 	ret = pagemap_ioctl(mem, mem_size, vec, vec_size,  in sanity_tests_sd()
364 	ret2 = pagemap_ioctl(mem, mem_size, vec, vec_size,  in sanity_tests_sd()
376 	memset(mem, -1, mem_size);  in sanity_tests_sd()
379 	ret = pagemap_ioctl(mem + page_size, 2 * page_size, vec, 1,  in sanity_tests_sd()
385 	ret2 = pagemap_ioctl(mem, mem_size, vec2, vec_size, 0, 0,  in sanity_tests_sd()
391 			 vec[0].start == (uintptr_t)(mem + page_size) &&  in sanity_tests_sd()
392 			 ret2 == 2 && LEN(vec2[0]) == 1 && vec2[0].start == (uintptr_t)mem &&  in sanity_tests_sd()
394 			 vec2[1].start == (uintptr_t)(mem + 3 * page_size),  in sanity_tests_sd()
397 	wp_free(mem, mem_size);  in sanity_tests_sd()
398 	munmap(mem, mem_size);  in sanity_tests_sd()
442 	mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0);  in sanity_tests_sd()
443 	if (mem == MAP_FAILED)  in sanity_tests_sd()
446 	wp_init(mem, mem_size);  in sanity_tests_sd()
447 	wp_addr_range(mem, mem_size);  in sanity_tests_sd()
449 	ret = pagemap_ioctl(mem, mem_size, vec, vec_size,  in sanity_tests_sd()
456 		mem[i * page_size]++;  in sanity_tests_sd()
458 	ret = pagemap_ioctl(mem, mem_size, vec, vec_size,  in sanity_tests_sd()
466 	ret = pagemap_ioctl(mem, mem_size, vec, vec_size,  in sanity_tests_sd()
474 	ret = pagemap_ioctl(mem, mem_size, vec, vec_size,  in sanity_tests_sd()
485 	wp_free(mem, mem_size);  in sanity_tests_sd()
486 	munmap(mem, mem_size);  in sanity_tests_sd()
497 	mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0);  in sanity_tests_sd()
498 	if (mem == MAP_FAILED)  in sanity_tests_sd()
501 	wp_init(mem, mem_size);  in sanity_tests_sd()
502 	wp_addr_range(mem, mem_size);  in sanity_tests_sd()
506 		walk_end = (long)mem;  in sanity_tests_sd()
510 				mem[i]++;  in sanity_tests_sd()
516 			ret = pagemap_ioc((void *)walk_end, mem_size-(walk_end - (long)mem), vec,  in sanity_tests_sd()
536 	wp_free(mem, mem_size);  in sanity_tests_sd()
537 	munmap(mem, mem_size);  in sanity_tests_sd()
547 	mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0);  in sanity_tests_sd()
548 	if (mem == MAP_FAILED)  in sanity_tests_sd()
551 	wp_init(mem, mem_size);  in sanity_tests_sd()
552 	wp_addr_range(mem, mem_size);  in sanity_tests_sd()
554 	memset(mem, 0, mem_size);  in sanity_tests_sd()
556 	ret = pagemap_ioc(mem, 0, vec, vec_size, 0,  in sanity_tests_sd()
560 	ksft_test_result(ret == 0 && walk_end == (long)mem,  in sanity_tests_sd()
563 	ret = pagemap_ioc(mem, 0, vec, vec_size, PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC,  in sanity_tests_sd()
567 	ksft_test_result(ret == 0 && walk_end == (long)mem,  in sanity_tests_sd()
570 	ret = pagemap_ioc(mem, 0, vec, 0, PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC,  in sanity_tests_sd()
574 	ksft_test_result(ret == 0 && walk_end == (long)mem,  in sanity_tests_sd()
577 	ret = pagemap_ioc(mem, mem_size, vec, vec_size, 0,  in sanity_tests_sd()
581 	ksft_test_result(ret == 1 && walk_end == (long)(mem + mem_size),  in sanity_tests_sd()
584 	ret = pagemap_ioc(mem, mem_size, vec, 1, 0,  in sanity_tests_sd()
588 	ksft_test_result(ret == 1 && walk_end == (long)(mem + mem_size),  in sanity_tests_sd()
591 	ret = pagemap_ioc(mem, mem_size, vec, 1, 0,  in sanity_tests_sd()
595 	ksft_test_result(ret == 1 && walk_end == (long)(mem + mem_size),  in sanity_tests_sd()
598 	ret = pagemap_ioc(mem, mem_size, vec, vec_size, 0,  in sanity_tests_sd()
602 	ksft_test_result(ret == 1 && walk_end == (long)(mem + mem_size/2),  in sanity_tests_sd()
605 	ret = pagemap_ioc(mem, mem_size, vec, vec_size, 0,  in sanity_tests_sd()
609 	ksft_test_result(ret == 1 && walk_end == (long)(mem + page_size),  in sanity_tests_sd()
612 	ret = pagemap_ioc(mem, mem_size, vec, vec_size, 0,  in sanity_tests_sd()
616 	ksft_test_result(ret == 1 && walk_end == (long)(mem + mem_size),  in sanity_tests_sd()
619 	wp_addr_range(mem, mem_size);  in sanity_tests_sd()
621 		mem[i]++;  in sanity_tests_sd()
623 	ret = pagemap_ioc(mem, mem_size, vec, vec_size, 0,  in sanity_tests_sd()
627 	ksft_test_result((unsigned long)ret == vec_size/2 && walk_end == (long)(mem + mem_size),  in sanity_tests_sd()
630 	ret = pagemap_ioc(mem, mem_size, vec, 1, 0,  in sanity_tests_sd()
634 	ksft_test_result(ret == 1 && walk_end == (long)(mem + page_size * 2),  in sanity_tests_sd()
637 	ret = pagemap_ioc(mem, mem_size, vec, 1, 0,  in sanity_tests_sd()
641 	ksft_test_result(ret == 1 && walk_end == (long)(mem + page_size * 2),  in sanity_tests_sd()
644 	ret = pagemap_ioc(mem, mem_size, vec, vec_size/2, 0,  in sanity_tests_sd()
648 	ksft_test_result((unsigned long)ret == vec_size/2 && walk_end == (long)(mem + mem_size),  in sanity_tests_sd()
651 	ret = pagemap_ioc(mem, mem_size, vec, vec_size, 0,  in sanity_tests_sd()
655 	ksft_test_result((unsigned long)ret == vec_size/2 && walk_end == (long)(mem + mem_size),  in sanity_tests_sd()
658 	ret = pagemap_ioc(mem, mem_size, vec, vec_size, 0,  in sanity_tests_sd()
662 	ksft_test_result((unsigned long)ret == vec_size/2 && walk_end == (long)(mem + mem_size),  in sanity_tests_sd()
665 	ret = pagemap_ioc(mem, mem_size, vec, vec_size, 0,  in sanity_tests_sd()
669 	ksft_test_result(ret == 1 && walk_end == (long)(mem + page_size * 2),  in sanity_tests_sd()
673 	wp_free(mem, mem_size);  in sanity_tests_sd()
674 	munmap(mem, mem_size);  in sanity_tests_sd()
679 int base_tests(char *prefix, char *mem, unsigned long long mem_size, int skip)  in base_tests()  argument
701 	written = pagemap_ioctl(mem, mem_size, vec, 1, PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC,  in base_tests()
709 	memset(mem, -1, mem_size);  in base_tests()
711 	written = pagemap_ioctl(mem, mem_size, vec, 1, 0, 0, PAGE_IS_WRITTEN, 0, 0,  in base_tests()
720 	written = pagemap_ioctl(mem, mem_size, vec, 1, PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC,  in base_tests()
725 	memset(mem + page_size, 0, mem_size - (2 * page_size));  in base_tests()
727 	written = pagemap_ioctl(mem, mem_size, vec, 1, PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC,  in base_tests()
735 	written = pagemap_ioctl(mem, mem_size, vec, 1, 0, 0,  in base_tests()
744 	written = pagemap_ioctl(mem, mem_size, vec, 1, PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC,  in base_tests()
749 	mem[vec_size/2 * page_size]++;  in base_tests()
751 	written = pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0, PAGE_IS_WRITTEN,  in base_tests()
760 	written = pagemap_ioctl(mem, mem_size, vec, 1, PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC,  in base_tests()
765 	mem[vec_size/2 * page_size]++;  in base_tests()
766 	mem[(vec_size/2 + 1) * page_size]++;  in base_tests()
768 	written = pagemap_ioctl(&mem[vec_size/2 * page_size], 2 * page_size, vec, 1, 0,  in base_tests()
773 	ksft_test_result(written == 1 && vec[0].start == (uintptr_t)(&mem[vec_size/2 * page_size])  in base_tests()
1056 	char *mem, *fmem;  in sanity_tests()  local
1065 	mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0);  in sanity_tests()
1066 	if (mem == MAP_FAILED || vec == MAP_FAILED)  in sanity_tests()
1069 	wp_init(mem, mem_size);  in sanity_tests()
1070 	wp_addr_range(mem, mem_size);  in sanity_tests()
1072 	ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size,  in sanity_tests()
1076 	ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0,  in sanity_tests()
1079 	ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0,  in sanity_tests()
1082 	ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0,  in sanity_tests()
1085 	ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0,  in sanity_tests()
1089 	wp_free(mem, mem_size);  in sanity_tests()
1090 	munmap(mem, mem_size);  in sanity_tests()
1093 	mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0);  in sanity_tests()
1094 	if (mem == MAP_FAILED)  in sanity_tests()
1096 	wp_init(mem, mem_size);  in sanity_tests()
1097 	wp_addr_range(mem, mem_size);  in sanity_tests()
1099 	memset(mem, 0, mem_size);  in sanity_tests()
1101 	ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0,  in sanity_tests()
1103 	ksft_test_result(ret >= 0 && vec[0].start == (uintptr_t)mem && LEN(vec[0]) == vec_size &&  in sanity_tests()
1109 	ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0,  in sanity_tests()
1111 	ksft_test_result(ret >= 0 && vec[0].start == (uintptr_t)mem && LEN(vec[0]) == vec_size &&  in sanity_tests()
1117 	ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0,  in sanity_tests()
1119 	ksft_test_result(ret >= 0 && vec[0].start == (uintptr_t)mem && LEN(vec[0]) == vec_size &&  in sanity_tests()
1126 	ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0,  in sanity_tests()
1131 	ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0,  in sanity_tests()
1135 	wp_free(mem, mem_size);  in sanity_tests()
1136 	munmap(mem, mem_size);  in sanity_tests()
1139 	mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0);  in sanity_tests()
1140 	if (mem == MAP_FAILED)  in sanity_tests()
1142 	wp_init(mem, mem_size);  in sanity_tests()
1143 	wp_addr_range(mem, mem_size);  in sanity_tests()
1145 	memset(mem, 0, mem_size);  in sanity_tests()
1147 	ret = pagemap_ioctl(mem, mem_size, vec, vec_size,  in sanity_tests()
1150 	ksft_test_result(ret >= 0 && vec[0].start == (uintptr_t)mem && LEN(vec[0]) == vec_size &&  in sanity_tests()
1153 	wp_free(mem, mem_size);  in sanity_tests()
1154 	munmap(mem, mem_size);  in sanity_tests()
1227 	char *mem, *mem2;  in mprotect_tests()  local
1237 	mem = mmap(0, 2 * page_size, PROT_READ|PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0);  in mprotect_tests()
1238 	if (mem == MAP_FAILED)  in mprotect_tests()
1240 	wp_init(mem, 2 * page_size);  in mprotect_tests()
1241 	wp_addr_range(mem, 2 * page_size);  in mprotect_tests()
1244 	memset(mem, 1, 2 * page_size);  in mprotect_tests()
1246 	ret = pagemap_ioctl(mem, 2 * page_size, &vec, 1, 0, 0, PAGE_IS_WRITTEN,  in mprotect_tests()
1254 	wp_addr_range(mem, 2 * page_size);  in mprotect_tests()
1256 	ksft_test_result(pagemap_ioctl(mem, 2 * page_size, &vec, 1, 0, 0,  in mprotect_tests()
1261 	mem2 = mmap(mem + page_size, page_size, PROT_READ|PROT_WRITE,  in mprotect_tests()
1269 	mprotect(mem, page_size, PROT_NONE);  in mprotect_tests()
1270 	mprotect(mem, 2 * page_size, PROT_READ);  in mprotect_tests()
1271 	mprotect(mem, 2 * page_size, PROT_READ|PROT_WRITE);  in mprotect_tests()
1274 	memset(mem, 2, 2 * page_size);  in mprotect_tests()
1277 	mprotect(mem, page_size, PROT_NONE);  in mprotect_tests()
1278 	mprotect(mem, page_size, PROT_READ);  in mprotect_tests()
1279 	mprotect(mem, page_size, PROT_READ|PROT_WRITE);  in mprotect_tests()
1281 	ret = pagemap_ioctl(mem, 2 * page_size, &vec, 1, 0, 0, PAGE_IS_WRITTEN,  in mprotect_tests()
1290 	wp_addr_range(mem, 2 * page_size);  in mprotect_tests()
1292 	memset(mem, 'A', 2 * page_size);  in mprotect_tests()
1294 	ret = pagemap_ioctl(mem, 2 * page_size, &vec, 1, 0, 0, PAGE_IS_WRITTEN,  in mprotect_tests()
1302 	wp_free(mem, 2 * page_size);  in mprotect_tests()
1303 	munmap(mem, 2 * page_size);  in mprotect_tests()
1314 static ssize_t get_dirty_pages_reset(char *mem, unsigned int count,  in get_dirty_pages_reset()  argument
1324 	arg.start = (uintptr_t)mem;  in get_dirty_pages_reset()
1326 	arg.end = (uintptr_t)(mem + count * page_size);  in get_dirty_pages_reset()
1350 void *thread_proc(void *mem)  in thread_proc()  argument
1352 	int *m = mem;  in thread_proc()
1395 	char *mem;  in transact_test()  local
1404 	mem = mmap(NULL, 0x1000 * nthreads * pages_per_thread, PROT_READ | PROT_WRITE,  in transact_test()
1406 	if (mem == MAP_FAILED)  in transact_test()
1409 	wp_init(mem, 0x1000 * nthreads * pages_per_thread);  in transact_test()
1410 	wp_addr_range(mem, 0x1000 * nthreads * pages_per_thread);  in transact_test()
1412 	memset(mem, 0, 0x1000 * nthreads * pages_per_thread);  in transact_test()
1414 	count = get_dirty_pages_reset(mem, nthreads * pages_per_thread, 1, page_size);  in transact_test()
1416 	count = get_dirty_pages_reset(mem, nthreads * pages_per_thread, 1, page_size);  in transact_test()
1421 		pthread_create(&th, NULL, thread_proc, mem + 0x1000 * i * pages_per_thread);  in transact_test()
1431 		count = get_dirty_pages_reset(mem, nthreads * pages_per_thread, 1,  in transact_test()
1442 		c = get_dirty_pages_reset(mem, nthreads * pages_per_thread, 1, page_size);  in transact_test()
1487 	char *mem, *map, *fmem;  in main()  local
1511 	mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0);  in main()
1512 	if (mem == MAP_FAILED)  in main()
1514 	wp_init(mem, mem_size);  in main()
1515 	wp_addr_range(mem, mem_size);  in main()
1517 	base_tests("Page testing:", mem, mem_size, 0);  in main()
1519 	wp_free(mem, mem_size);  in main()
1520 	munmap(mem, mem_size);  in main()
1524 	mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0);  in main()
1525 	if (mem == MAP_FAILED)  in main()
1527 	wp_init(mem, mem_size);  in main()
1528 	wp_addr_range(mem, mem_size);  in main()
1530 	base_tests("Large Page testing:", mem, mem_size, 0);  in main()
1532 	wp_free(mem, mem_size);  in main()
1533 	munmap(mem, mem_size);  in main()
1549 	mem = gethugetlb_mem(mem_size, &shmid);  in main()
1550 	if (mem) {  in main()
1551 		wp_init(mem, mem_size);  in main()
1552 		wp_addr_range(mem, mem_size);  in main()
1554 		base_tests("Hugetlb shmem testing:", mem, mem_size, 0);  in main()
1556 		wp_free(mem, mem_size);  in main()
1563 	mem = gethugetlb_mem(mem_size, NULL);  in main()
1564 	if (mem) {  in main()
1565 		wp_init(mem, mem_size);  in main()
1566 		wp_addr_range(mem, mem_size);  in main()
1568 		base_tests("Hugetlb mem testing:", mem, mem_size, 0);  in main()
1570 		wp_free(mem, mem_size);  in main()
1572 		base_tests("Hugetlb mem testing:", NULL, 0, 1);  in main()
1580 	mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);  in main()
1581 	if (mem != MAP_FAILED) {  in main()
1582 		wp_init(mem, mem_size);  in main()
1583 		wp_addr_range(mem, mem_size);  in main()
1585 		base_tests("Hugetlb shmem testing:", mem, mem_size, 0);  in main()
1587 		wp_free(mem, mem_size);  in main()