17 size_t copy_to_user_iter(void __user *iter_to, size_t progress,  in copy_to_user_iter()
18 			 size_t len, void *from, void *priv2)  in copy_to_user_iter()
31 size_t copy_to_user_iter_nofault(void __user *iter_to, size_t progress,  in copy_to_user_iter_nofault()
32 				 size_t len, void *from, void *priv2)  in copy_to_user_iter_nofault()
45 size_t copy_from_user_iter(void __user *iter_from, size_t progress,  in copy_from_user_iter()
46 			   size_t len, void *to, void *priv2)  in copy_from_user_iter()
48 	size_t res = len;  in copy_from_user_iter()
62 size_t memcpy_to_iter(void *iter_to, size_t progress,  in memcpy_to_iter()
63 		      size_t len, void *from, void *priv2)  in memcpy_to_iter()
70 size_t memcpy_from_iter(void *iter_from, size_t progress,  in memcpy_from_iter()
71 			size_t len, void *to, void *priv2)  in memcpy_from_iter()
90 size_t fault_in_iov_iter_readable(const struct iov_iter *i, size_t size)  in fault_in_iov_iter_readable()
93 		size_t n = min(size, iov_iter_count(i));  in fault_in_iov_iter_readable()
97 		size_t count = min(size, iov_iter_count(i));  in fault_in_iov_iter_readable()
99 		size_t skip;  in fault_in_iov_iter_readable()
103 			size_t len = min(count, p->iov_len - skip);  in fault_in_iov_iter_readable()
104 			size_t ret;  in fault_in_iov_iter_readable()
133 size_t fault_in_iov_iter_writeable(const struct iov_iter *i, size_t size)  in fault_in_iov_iter_writeable()
136 		size_t n = min(size, iov_iter_count(i));  in fault_in_iov_iter_writeable()
140 		size_t count = min(size, iov_iter_count(i));  in fault_in_iov_iter_writeable()
142 		size_t skip;  in fault_in_iov_iter_writeable()
146 			size_t len = min(count, p->iov_len - skip);  in fault_in_iov_iter_writeable()
147 			size_t ret;  in fault_in_iov_iter_writeable()
164 			size_t count)  in iov_iter_init()
179 size_t _copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i)  in _copy_to_iter()
192 size_t copy_to_user_iter_mc(void __user *iter_to, size_t progress,  in copy_to_user_iter_mc()
193 			    size_t len, void *from, void *priv2)  in copy_to_user_iter_mc()
204 size_t memcpy_to_iter_mc(void *iter_to, size_t progress,  in memcpy_to_iter_mc()
205 			 size_t len, void *from, void *priv2)  in memcpy_to_iter_mc()
234 size_t _copy_mc_to_iter(const void *addr, size_t bytes, struct iov_iter *i)  in _copy_mc_to_iter()
247 size_t __copy_from_iter(void *addr, size_t bytes, struct iov_iter *i)  in __copy_from_iter()
253 size_t _copy_from_iter(void *addr, size_t bytes, struct iov_iter *i)  in _copy_from_iter()
265 size_t copy_from_user_iter_nocache(void __user *iter_from, size_t progress,  in copy_from_user_iter_nocache()
266 				   size_t len, void *to, void *priv2)  in copy_from_user_iter_nocache()
271 size_t _copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i)  in _copy_from_iter_nocache()
284 size_t copy_from_user_iter_flushcache(void __user *iter_from, size_t progress,  in copy_from_user_iter_flushcache()
285 				      size_t len, void *to, void *priv2)  in copy_from_user_iter_flushcache()
291 size_t memcpy_from_iter_flushcache(void *iter_from, size_t progress,  in memcpy_from_iter_flushcache()
292 				   size_t len, void *to, void *priv2)  in memcpy_from_iter_flushcache()
314 size_t _copy_from_iter_flushcache(void *addr, size_t bytes, struct iov_iter *i)  in _copy_from_iter_flushcache()
326 static inline bool page_copy_sane(struct page *page, size_t offset, size_t n)  in page_copy_sane()
329 	size_t v = n + offset;  in page_copy_sane()
349 size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes,  in copy_page_to_iter()
352 	size_t res = 0;  in copy_page_to_iter()
361 		size_t n = min(bytes, (size_t)PAGE_SIZE - offset);  in copy_page_to_iter()
378 size_t copy_page_to_iter_nofault(struct page *page, unsigned offset, size_t bytes,  in copy_page_to_iter_nofault()
381 	size_t res = 0;  in copy_page_to_iter_nofault()
391 		size_t n = min(bytes, (size_t)PAGE_SIZE - offset);  in copy_page_to_iter_nofault()
411 size_t copy_page_from_iter(struct page *page, size_t offset, size_t bytes,  in copy_page_from_iter()
414 	size_t res = 0;  in copy_page_from_iter()
421 		size_t n = min(bytes, (size_t)PAGE_SIZE - offset);  in copy_page_from_iter()
439 size_t zero_to_user_iter(void __user *iter_to, size_t progress,  in zero_to_user_iter()
440 			 size_t len, void *priv, void *priv2)  in zero_to_user_iter()
446 size_t zero_to_iter(void *iter_to, size_t progress,  in zero_to_iter()
447 		    size_t len, void *priv, void *priv2)  in zero_to_iter()
453 size_t iov_iter_zero(size_t bytes, struct iov_iter *i)  in iov_iter_zero()
460 size_t copy_page_from_iter_atomic(struct page *page, size_t offset,  in copy_page_from_iter_atomic()
461 		size_t bytes, struct iov_iter *i)  in copy_page_from_iter_atomic()
463 	size_t n, copied = 0;  in copy_page_from_iter_atomic()
479 			n = min_t(size_t, n, PAGE_SIZE - offset);  in copy_page_from_iter_atomic()
493 static void iov_iter_bvec_advance(struct iov_iter *i, size_t size)  in iov_iter_bvec_advance()
513 static void iov_iter_iovec_advance(struct iov_iter *i, size_t size)  in iov_iter_iovec_advance()
532 static void iov_iter_folioq_advance(struct iov_iter *i, size_t size)  in iov_iter_folioq_advance()
548 		size_t fsize = folioq_folio_size(folioq, slot);  in iov_iter_folioq_advance()
565 void iov_iter_advance(struct iov_iter *i, size_t size)  in iov_iter_advance()
585 static void iov_iter_folioq_revert(struct iov_iter *i, size_t unroll)  in iov_iter_folioq_revert()
591 		size_t fsize;  in iov_iter_folioq_revert()
611 void iov_iter_revert(struct iov_iter *i, size_t unroll)  in iov_iter_revert()
633 			size_t n = (--bvec)->bv_len;  in iov_iter_revert()
648 			size_t n = (--iov)->iov_len;  in iov_iter_revert()
664 size_t iov_iter_single_seg_count(const struct iov_iter *i)  in iov_iter_single_seg_count()
681 			size_t count)  in iov_iter_kvec()
697 			size_t count)  in iov_iter_bvec()
727 			  unsigned int offset, size_t count)  in iov_iter_folio_queue()
755 		     struct xarray *xarray, loff_t start, size_t count)  in iov_iter_xarray()
778 void iov_iter_discard(struct iov_iter *i, unsigned int direction, size_t count)  in iov_iter_discard()
794 	size_t size = i->count;  in iov_iter_aligned_iovec()
795 	size_t skip = i->iov_offset;  in iov_iter_aligned_iovec()
798 		size_t len = iov->iov_len - skip;  in iov_iter_aligned_iovec()
820 	size_t size = i->count;  in iov_iter_aligned_bvec()
823 		size_t len = bvec->bv_len;  in iov_iter_aligned_bvec()
889 	size_t size = i->count;  in iov_iter_alignment_iovec()
890 	size_t skip = i->iov_offset;  in iov_iter_alignment_iovec()
893 		size_t len = iov->iov_len - skip;  in iov_iter_alignment_iovec()
911 	size_t size = i->count;  in iov_iter_alignment_bvec()
915 		size_t len = bvec->bv_len - skip;  in iov_iter_alignment_bvec()
931 		size_t size = i->count;  in iov_iter_alignment()
958 	size_t size = i->count;  in iov_iter_gap_alignment()
983 static int want_pages_array(struct page ***res, size_t size,  in want_pages_array()
984 			    size_t start, unsigned int maxpages)  in want_pages_array()
1000 				     struct page ***ppages, size_t maxsize,  in iter_folioq_get_pages()
1001 				     unsigned maxpages, size_t *_start_offset)  in iter_folioq_get_pages()
1006 	size_t extracted = 0, count = iter->count, iov_offset = iter->iov_offset;  in iter_folioq_get_pages()
1023 		size_t offset = iov_offset, fsize = folioq_folio_size(folioq, slot);  in iter_folioq_get_pages()
1024 		size_t part = PAGE_SIZE - offset % PAGE_SIZE;  in iter_folioq_get_pages()
1086 				     struct page ***pages, size_t maxsize,  in iter_xarray_get_pages()
1087 				     unsigned maxpages, size_t *_start_offset)  in iter_xarray_get_pages()
1105 	maxsize = min_t(size_t, nr * PAGE_SIZE - offset, maxsize);  in iter_xarray_get_pages()
1112 static unsigned long first_iovec_segment(const struct iov_iter *i, size_t *size)  in first_iovec_segment()
1114 	size_t skip;  in first_iovec_segment()
1122 		size_t len = iov->iov_len - skip;  in first_iovec_segment()
1135 				       size_t *size, size_t *start)  in first_bvec_segment()
1138 	size_t skip = i->iov_offset, len;  in first_bvec_segment()
1150 		   struct page ***pages, size_t maxsize,  in __iov_iter_get_pages_alloc()
1151 		   unsigned int maxpages, size_t *start)  in __iov_iter_get_pages_alloc()
1180 		maxsize = min_t(size_t, maxsize, res * PAGE_SIZE - *start);  in __iov_iter_get_pages_alloc()
1199 		maxsize = min_t(size_t, maxsize, n * PAGE_SIZE - *start);  in __iov_iter_get_pages_alloc()
1217 		size_t maxsize, unsigned maxpages, size_t *start)  in iov_iter_get_pages2()
1228 		struct page ***pages, size_t maxsize, size_t *start)  in iov_iter_get_pages_alloc2()
1245 	size_t skip = i->iov_offset, size = i->count;  in iov_npages()
1251 		size_t len = min(p->iov_len - skip, size);  in iov_npages()
1265 	size_t skip = i->iov_offset, size = i->count;  in bvec_npages()
1271 		size_t len = min(p->bv_len - skip, size);  in bvec_npages()
1531 int import_ubuf(int rw, void __user *buf, size_t len, struct iov_iter *i)  in import_ubuf()
1586 					     struct page ***pages, size_t maxsize,  in iov_iter_extract_folioq_pages()
1589 					     size_t *offset0)  in iov_iter_extract_folioq_pages()
1594 	size_t extracted = 0, offset, slot = i->folioq_slot;  in iov_iter_extract_folioq_pages()
1613 		size_t offset = i->iov_offset, fsize = folioq_folio_size(folioq, slot);  in iov_iter_extract_folioq_pages()
1614 		size_t part = PAGE_SIZE - offset % PAGE_SIZE;  in iov_iter_extract_folioq_pages()
1648 					     struct page ***pages, size_t maxsize,  in iov_iter_extract_xarray_pages()
1651 					     size_t *offset0)  in iov_iter_extract_xarray_pages()
1684 	maxsize = min_t(size_t, nr * PAGE_SIZE - offset, maxsize);  in iov_iter_extract_xarray_pages()
1694 					   struct page ***pages, size_t maxsize,  in iov_iter_extract_bvec_pages()
1697 					   size_t *offset0)  in iov_iter_extract_bvec_pages()
1699 	size_t skip = i->iov_offset, size = 0;  in iov_iter_extract_bvec_pages()
1761 					   struct page ***pages, size_t maxsize,  in iov_iter_extract_kvec_pages()
1764 					   size_t *offset0)  in iov_iter_extract_kvec_pages()
1768 	size_t skip = i->iov_offset, offset, len, size;  in iov_iter_extract_kvec_pages()
1795 		size_t seg = min_t(size_t, len, PAGE_SIZE);  in iov_iter_extract_kvec_pages()
1807 	size = min_t(size_t, size, maxpages * PAGE_SIZE - offset);  in iov_iter_extract_kvec_pages()
1826 					   size_t maxsize,  in iov_iter_extract_user_pages()
1829 					   size_t *offset0)  in iov_iter_extract_user_pages()
1833 	size_t offset;  in iov_iter_extract_user_pages()
1852 	maxsize = min_t(size_t, maxsize, res * PAGE_SIZE - offset);  in iov_iter_extract_user_pages()
1902 			       size_t maxsize,  in iov_iter_extract_pages()
1905 			       size_t *offset0)  in iov_iter_extract_pages()
1907 	maxsize = min_t(size_t, min_t(size_t, maxsize, i->count), MAX_RW_COUNT);  in iov_iter_extract_pages()