Lines Matching refs:vma
81 void vma_set_page_prot(struct vm_area_struct *vma) in vma_set_page_prot() argument
83 unsigned long vm_flags = vma->vm_flags; in vma_set_page_prot()
86 vm_page_prot = vm_pgprot_modify(vma->vm_page_prot, vm_flags); in vma_set_page_prot()
87 if (vma_wants_writenotify(vma, vm_page_prot)) { in vma_set_page_prot()
92 WRITE_ONCE(vma->vm_page_prot, vm_page_prot); in vma_set_page_prot()
696 struct vm_area_struct *vma, *prev; in generic_get_unmapped_area() local
708 vma = find_vma_prev(mm, addr, &prev); in generic_get_unmapped_area()
710 (!vma || addr + len <= vm_start_gap(vma)) && in generic_get_unmapped_area()
744 struct vm_area_struct *vma, *prev; in generic_get_unmapped_area_topdown() local
759 vma = find_vma_prev(mm, addr, &prev); in generic_get_unmapped_area_topdown()
761 (!vma || addr + len <= vm_start_gap(vma)) && in generic_get_unmapped_area_topdown()
934 struct vm_area_struct *vma; in find_vma_prev() local
937 vma = vma_iter_load(&vmi); in find_vma_prev()
939 if (!vma) in find_vma_prev()
940 vma = vma_next(&vmi); in find_vma_prev()
941 return vma; in find_vma_prev()
961 int expand_stack_locked(struct vm_area_struct *vma, unsigned long address) in expand_stack_locked() argument
963 return expand_upwards(vma, address); in expand_stack_locked()
968 struct vm_area_struct *vma, *prev; in find_extend_vma_locked() local
971 vma = find_vma_prev(mm, addr, &prev); in find_extend_vma_locked()
972 if (vma && (vma->vm_start <= addr)) in find_extend_vma_locked()
973 return vma; in find_extend_vma_locked()
983 int expand_stack_locked(struct vm_area_struct *vma, unsigned long address) in expand_stack_locked() argument
985 return expand_downwards(vma, address); in expand_stack_locked()
990 struct vm_area_struct *vma; in find_extend_vma_locked() local
994 vma = find_vma(mm, addr); in find_extend_vma_locked()
995 if (!vma) in find_extend_vma_locked()
997 if (vma->vm_start <= addr) in find_extend_vma_locked()
998 return vma; in find_extend_vma_locked()
999 start = vma->vm_start; in find_extend_vma_locked()
1000 if (expand_stack_locked(vma, addr)) in find_extend_vma_locked()
1002 if (vma->vm_flags & VM_LOCKED) in find_extend_vma_locked()
1003 populate_vma_page_range(vma, addr, start, NULL); in find_extend_vma_locked()
1004 return vma; in find_extend_vma_locked()
1010 #define vma_expand_up(vma,addr) expand_upwards(vma, addr) argument
1011 #define vma_expand_down(vma, addr) (-EFAULT) argument
1015 #define vma_expand_up(vma,addr) (-EFAULT) argument
1016 #define vma_expand_down(vma, addr) expand_downwards(vma, addr) argument
1033 struct vm_area_struct *vma, *prev; in expand_stack() local
1039 vma = find_vma_prev(mm, addr, &prev); in expand_stack()
1040 if (vma && vma->vm_start <= addr) in expand_stack()
1044 vma = prev; in expand_stack()
1048 if (vma && !vma_expand_down(vma, addr)) in expand_stack()
1056 return vma; in expand_stack()
1096 struct vm_area_struct *vma; in SYSCALL_DEFINE5() local
1125 vma = vma_lookup(mm, start); in SYSCALL_DEFINE5()
1127 if (!vma || !(vma->vm_flags & VM_SHARED)) { in SYSCALL_DEFINE5()
1132 prot |= vma->vm_flags & VM_READ ? PROT_READ : 0; in SYSCALL_DEFINE5()
1133 prot |= vma->vm_flags & VM_WRITE ? PROT_WRITE : 0; in SYSCALL_DEFINE5()
1134 prot |= vma->vm_flags & VM_EXEC ? PROT_EXEC : 0; in SYSCALL_DEFINE5()
1138 if (vma->vm_flags & VM_LOCKED) in SYSCALL_DEFINE5()
1142 vm_flags = vma->vm_flags; in SYSCALL_DEFINE5()
1143 file = get_file(vma->vm_file); in SYSCALL_DEFINE5()
1162 vma = vma_lookup(mm, start); in SYSCALL_DEFINE5()
1164 if (!vma) in SYSCALL_DEFINE5()
1168 if (vma->vm_flags != vm_flags) in SYSCALL_DEFINE5()
1170 if (vma->vm_file != file) in SYSCALL_DEFINE5()
1173 if (start + size > vma->vm_end) { in SYSCALL_DEFINE5()
1174 VMA_ITERATOR(vmi, mm, vma->vm_end); in SYSCALL_DEFINE5()
1175 struct vm_area_struct *next, *prev = vma; in SYSCALL_DEFINE5()
1182 if (next->vm_file != vma->vm_file) in SYSCALL_DEFINE5()
1185 if (next->vm_flags != vma->vm_flags) in SYSCALL_DEFINE5()
1198 ret = do_mmap(vma->vm_file, start, size, in SYSCALL_DEFINE5()
1213 struct vm_area_struct *vma = NULL; in vm_brk_flags() local
1241 vma = vma_prev(&vmi); in vm_brk_flags()
1242 ret = do_brk_flags(&vmi, vma, addr, len, flags); in vm_brk_flags()
1261 struct vm_area_struct *vma; in exit_mmap() local
1272 vma = vma_next(&vmi); in exit_mmap()
1273 if (!vma || unlikely(xa_is_zero(vma))) { in exit_mmap()
1284 unmap_vmas(&tlb, &vmi.mas, vma, 0, ULONG_MAX, ULONG_MAX, false); in exit_mmap()
1294 vma_iter_set(&vmi, vma->vm_end); in exit_mmap()
1295 free_pgtables(&tlb, &vmi.mas, vma, FIRST_USER_ADDRESS, in exit_mmap()
1304 vma_iter_set(&vmi, vma->vm_end); in exit_mmap()
1306 if (vma->vm_flags & VM_ACCOUNT) in exit_mmap()
1307 nr_accounted += vma_pages(vma); in exit_mmap()
1308 remove_vma(vma, /* unreachable = */ true); in exit_mmap()
1311 vma = vma_next(&vmi); in exit_mmap()
1312 } while (vma && likely(!xa_is_zero(vma))); in exit_mmap()
1327 int insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma) in insert_vm_struct() argument
1329 unsigned long charged = vma_pages(vma); in insert_vm_struct()
1332 if (find_vma_intersection(mm, vma->vm_start, vma->vm_end)) in insert_vm_struct()
1335 if ((vma->vm_flags & VM_ACCOUNT) && in insert_vm_struct()
1351 if (vma_is_anonymous(vma)) { in insert_vm_struct()
1352 BUG_ON(vma->anon_vma); in insert_vm_struct()
1353 vma->vm_pgoff = vma->vm_start >> PAGE_SHIFT; in insert_vm_struct()
1356 if (vma_link(mm, vma)) { in insert_vm_struct()
1357 if (vma->vm_flags & VM_ACCOUNT) in insert_vm_struct()
1413 static void special_mapping_close(struct vm_area_struct *vma) in special_mapping_close() argument
1415 const struct vm_special_mapping *sm = vma->vm_private_data; in special_mapping_close()
1418 sm->close(sm, vma); in special_mapping_close()
1421 static const char *special_mapping_name(struct vm_area_struct *vma) in special_mapping_name() argument
1423 return ((struct vm_special_mapping *)vma->vm_private_data)->name; in special_mapping_name()
1439 static int special_mapping_split(struct vm_area_struct *vma, unsigned long addr) in special_mapping_split() argument
1462 struct vm_area_struct *vma = vmf->vma; in special_mapping_fault() local
1465 struct vm_special_mapping *sm = vma->vm_private_data; in special_mapping_fault()
1468 return sm->fault(sm, vmf->vma, vmf); in special_mapping_fault()
1492 struct vm_area_struct *vma; in __install_special_mapping() local
1494 vma = vm_area_alloc(mm); in __install_special_mapping()
1495 if (unlikely(vma == NULL)) in __install_special_mapping()
1498 vma_set_range(vma, addr, addr + len, 0); in __install_special_mapping()
1499 vm_flags_init(vma, (vm_flags | mm->def_flags | in __install_special_mapping()
1501 vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); in __install_special_mapping()
1503 vma->vm_ops = ops; in __install_special_mapping()
1504 vma->vm_private_data = priv; in __install_special_mapping()
1506 ret = insert_vm_struct(mm, vma); in __install_special_mapping()
1510 vm_stat_account(mm, vma->vm_flags, len >> PAGE_SHIFT); in __install_special_mapping()
1512 perf_event_mmap(vma); in __install_special_mapping()
1514 return vma; in __install_special_mapping()
1517 vm_area_free(vma); in __install_special_mapping()
1521 bool vma_is_special_mapping(const struct vm_area_struct *vma, in vma_is_special_mapping() argument
1524 return vma->vm_private_data == sm && in vma_is_special_mapping()
1525 vma->vm_ops == &special_mapping_vmops; in vma_is_special_mapping()
1673 int relocate_vma_down(struct vm_area_struct *vma, unsigned long shift) in relocate_vma_down() argument
1686 struct mm_struct *mm = vma->vm_mm; in relocate_vma_down()
1687 unsigned long old_start = vma->vm_start; in relocate_vma_down()
1688 unsigned long old_end = vma->vm_end; in relocate_vma_down()
1693 VMG_STATE(vmg, mm, &vmi, new_start, old_end, 0, vma->vm_pgoff); in relocate_vma_down()
1703 if (vma != vma_next(&vmi)) in relocate_vma_down()
1710 vmg.vma = vma; in relocate_vma_down()
1718 if (length != move_page_tables(vma, old_start, in relocate_vma_down()
1719 vma, new_start, length, false, true)) in relocate_vma_down()
1744 return vma_shrink(&vmi, vma, new_start, new_end, vma->vm_pgoff); in relocate_vma_down()
1792 bool mmap_read_lock_maybe_expand(struct mm_struct *mm, struct vm_area_struct *vma, in mmap_read_lock_maybe_expand() argument