1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
4 */
5 #include <linux/init.h>
6 #include <linux/export.h>
7 #include <linux/mm.h>
8 #include <asm/pgalloc.h>
9 #include <asm/pgtable.h>
10 #include <asm/tlbflush.h>
11
dmw_virt_to_page(unsigned long kaddr)12 struct page *dmw_virt_to_page(unsigned long kaddr)
13 {
14 return phys_to_page(__pa(kaddr));
15 }
16 EXPORT_SYMBOL(dmw_virt_to_page);
17
tlb_virt_to_page(unsigned long kaddr)18 struct page *tlb_virt_to_page(unsigned long kaddr)
19 {
20 return phys_to_page(pfn_to_phys(pte_pfn(*virt_to_kpte(kaddr))));
21 }
22 EXPORT_SYMBOL(tlb_virt_to_page);
23
pgd_alloc(struct mm_struct * mm)24 pgd_t *pgd_alloc(struct mm_struct *mm)
25 {
26 pgd_t *init, *ret;
27
28 ret = __pgd_alloc(mm, 0);
29 if (ret) {
30 init = pgd_offset(&init_mm, 0UL);
31 pgd_init(ret);
32 memcpy(ret + USER_PTRS_PER_PGD, init + USER_PTRS_PER_PGD,
33 (PTRS_PER_PGD - USER_PTRS_PER_PGD) * sizeof(pgd_t));
34 }
35
36 return ret;
37 }
38 EXPORT_SYMBOL_GPL(pgd_alloc);
39
pgd_init(void * addr)40 void pgd_init(void *addr)
41 {
42 unsigned long *p, *end;
43 unsigned long entry;
44
45 #if !defined(__PAGETABLE_PUD_FOLDED)
46 entry = (unsigned long)invalid_pud_table;
47 #elif !defined(__PAGETABLE_PMD_FOLDED)
48 entry = (unsigned long)invalid_pmd_table;
49 #else
50 entry = (unsigned long)invalid_pte_table;
51 #endif
52
53 p = (unsigned long *)addr;
54 end = p + PTRS_PER_PGD;
55
56 do {
57 p[0] = entry;
58 p[1] = entry;
59 p[2] = entry;
60 p[3] = entry;
61 p[4] = entry;
62 p += 8;
63 p[-3] = entry;
64 p[-2] = entry;
65 p[-1] = entry;
66 } while (p != end);
67 }
68 EXPORT_SYMBOL_GPL(pgd_init);
69
70 #ifndef __PAGETABLE_PMD_FOLDED
pmd_init(void * addr)71 void pmd_init(void *addr)
72 {
73 unsigned long *p, *end;
74 unsigned long pagetable = (unsigned long)invalid_pte_table;
75
76 p = (unsigned long *)addr;
77 end = p + PTRS_PER_PMD;
78
79 do {
80 p[0] = pagetable;
81 p[1] = pagetable;
82 p[2] = pagetable;
83 p[3] = pagetable;
84 p[4] = pagetable;
85 p += 8;
86 p[-3] = pagetable;
87 p[-2] = pagetable;
88 p[-1] = pagetable;
89 } while (p != end);
90 }
91 EXPORT_SYMBOL_GPL(pmd_init);
92 #endif
93
94 #ifndef __PAGETABLE_PUD_FOLDED
pud_init(void * addr)95 void pud_init(void *addr)
96 {
97 unsigned long *p, *end;
98 unsigned long pagetable = (unsigned long)invalid_pmd_table;
99
100 p = (unsigned long *)addr;
101 end = p + PTRS_PER_PUD;
102
103 do {
104 p[0] = pagetable;
105 p[1] = pagetable;
106 p[2] = pagetable;
107 p[3] = pagetable;
108 p[4] = pagetable;
109 p += 8;
110 p[-3] = pagetable;
111 p[-2] = pagetable;
112 p[-1] = pagetable;
113 } while (p != end);
114 }
115 EXPORT_SYMBOL_GPL(pud_init);
116 #endif
117
kernel_pte_init(void * addr)118 void kernel_pte_init(void *addr)
119 {
120 unsigned long *p, *end;
121
122 p = (unsigned long *)addr;
123 end = p + PTRS_PER_PTE;
124
125 do {
126 p[0] = _PAGE_GLOBAL;
127 p[1] = _PAGE_GLOBAL;
128 p[2] = _PAGE_GLOBAL;
129 p[3] = _PAGE_GLOBAL;
130 p[4] = _PAGE_GLOBAL;
131 p += 8;
132 p[-3] = _PAGE_GLOBAL;
133 p[-2] = _PAGE_GLOBAL;
134 p[-1] = _PAGE_GLOBAL;
135 } while (p != end);
136 }
137
mk_pmd(struct page * page,pgprot_t prot)138 pmd_t mk_pmd(struct page *page, pgprot_t prot)
139 {
140 pmd_t pmd;
141
142 pmd_val(pmd) = (page_to_pfn(page) << PFN_PTE_SHIFT) | pgprot_val(prot);
143
144 return pmd;
145 }
146
set_pmd_at(struct mm_struct * mm,unsigned long addr,pmd_t * pmdp,pmd_t pmd)147 void set_pmd_at(struct mm_struct *mm, unsigned long addr,
148 pmd_t *pmdp, pmd_t pmd)
149 {
150 WRITE_ONCE(*pmdp, pmd);
151 flush_tlb_all();
152 }
153
pagetable_init(void)154 void __init pagetable_init(void)
155 {
156 /* Initialize the entire pgd. */
157 pgd_init(swapper_pg_dir);
158 pgd_init(invalid_pg_dir);
159 #ifndef __PAGETABLE_PUD_FOLDED
160 pud_init(invalid_pud_table);
161 #endif
162 #ifndef __PAGETABLE_PMD_FOLDED
163 pmd_init(invalid_pmd_table);
164 #endif
165 }
166