1/*************************************************************************************** 2* Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences 3* Copyright (c) 2020-2021 Peng Cheng Laboratory 4* 5* XiangShan is licensed under Mulan PSL v2. 6* You can use this software according to the terms and conditions of the Mulan PSL v2. 7* You may obtain a copy of Mulan PSL v2 at: 8* http://license.coscl.org.cn/MulanPSL2 9 10* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, 11* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, 12* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. 13* 14* See the Mulan PSL v2 for more details. 15* 16* 17* Acknowledgement 18* 19* This implementation is inspired by several key papers: 20* [1] Binh Pham, Viswanathan Vaidyanathan, Aamer Jaleel, and Abhishek Bhattacharjee. "[Colt: Coalesced large-reach 21* tlbs.](https://doi.org/10.1109/MICRO.2012.32)" 45th Annual IEEE/ACM International Symposium on Microarchitecture 22* (MICRO). 2012. 23***************************************************************************************/ 24 25package xiangshan.cache.mmu 26 27import org.chipsalliance.cde.config.Parameters 28import chisel3._ 29import chisel3.util._ 30import difftest._ 31import freechips.rocketchip.util.SRAMAnnotation 32import xiangshan._ 33import utils._ 34import utility._ 35import xiangshan.backend.fu.{PMPChecker, PMPReqBundle, PMPConfig => XSPMPConfig} 36import xiangshan.backend.rob.RobPtr 37import xiangshan.backend.fu.util.HasCSRConst 38import freechips.rocketchip.rocket.PMPConfig 39 40/** TLB module 41 * support block request and non-block request io at the same time 42 * return paddr at next cycle, then go for pmp/pma check 43 * @param Width: The number of requestors 44 * @param Block: Blocked or not for each requestor ports 45 * @param q: TLB Parameters, like entry number, each TLB has its own parameters 46 * @param p: XiangShan Paramemters, like XLEN 47 */ 48 49class TLB(Width: Int, nRespDups: Int = 1, Block: Seq[Boolean], q: TLBParameters)(implicit p: Parameters) extends TlbModule 50 with HasCSRConst 51 with HasPerfEvents 52{ 53 val io = IO(new TlbIO(Width, nRespDups, q)) 54 55 val req = io.requestor.map(_.req) 56 val resp = io.requestor.map(_.resp) 57 val ptw = io.ptw 58 val pmp = io.pmp 59 val refill_to_mem = io.refill_to_mem 60 61 /** Sfence.vma & Svinval 62 * Sfence.vma will 1. flush old entries 2. flush inflight 3. flush pipe 63 * Svinval will 1. flush old entries 2. flush inflight 64 * So, Svinval will not flush pipe, which means 65 * it should not drop reqs from pipe and should return right resp 66 */ 67 val sfence = DelayN(io.sfence, q.fenceDelay) 68 val csr = DelayN(io.csr, q.fenceDelay) 69 70 val flush_mmu = sfence.valid || csr.satp.changed || csr.vsatp.changed || csr.hgatp.changed 71 val mmu_flush_pipe = sfence.valid && sfence.bits.flushPipe // for svinval, won't flush pipe 72 val flush_pipe = io.flushPipe 73 val redirect = io.redirect 74 val EffectiveVa = Wire(Vec(Width, UInt(XLEN.W))) 75 val req_in = req 76 val req_out = Reg(Vec(Width, new TlbReq)) 77 for (i <- 0 until Width) { 78 when (req(i).fire) { 79 req_out(i) := req(i).bits 80 req_out(i).fullva := EffectiveVa(i) 81 } 82 } 83 val req_out_v = (0 until Width).map(i => ValidHold(req_in(i).fire && !req_in(i).bits.kill, resp(i).fire, flush_pipe(i))) 84 85 val isHyperInst = (0 until Width).map(i => req_out_v(i) && req_out(i).hyperinst) 86 87 // ATTENTION: csr and flush from backend are delayed. csr should not be later than flush. 88 // because, csr will influence tlb behavior. 89 val ifetch = if (q.fetchi) true.B else false.B 90 val mode_tmp = if (q.useDmode) csr.priv.dmode else csr.priv.imode 91 val mode = (0 until Width).map(i => Mux(isHyperInst(i), csr.priv.spvp, mode_tmp)) 92 val virt_in = csr.priv.virt 93 val virt_out = req.map(a => RegEnable(csr.priv.virt, a.fire)) 94 val sum = (0 until Width).map(i => Mux(virt_out(i) || isHyperInst(i), csr.priv.vsum, csr.priv.sum)) 95 val mxr = (0 until Width).map(i => Mux(virt_out(i) || isHyperInst(i), csr.priv.vmxr || csr.priv.mxr, csr.priv.mxr)) 96 val req_in_s2xlate = (0 until Width).map(i => MuxCase(noS2xlate, Seq( 97 (!(virt_in || req_in(i).bits.hyperinst)) -> noS2xlate, 98 (csr.vsatp.mode =/= 0.U && csr.hgatp.mode =/= 0.U) -> allStage, 99 (csr.vsatp.mode === 0.U) -> onlyStage2, 100 (csr.hgatp.mode === 0.U) -> onlyStage1 101 ))) 102 val req_out_s2xlate = (0 until Width).map(i => MuxCase(noS2xlate, Seq( 103 (!(virt_out(i) || isHyperInst(i))) -> noS2xlate, 104 (csr.vsatp.mode =/= 0.U && csr.hgatp.mode =/= 0.U) -> allStage, 105 (csr.vsatp.mode === 0.U) -> onlyStage2, 106 (csr.hgatp.mode === 0.U) -> onlyStage1 107 ))) 108 val need_gpa = RegInit(false.B) 109 val need_gpa_wire = WireInit(false.B) 110 val need_gpa_robidx = Reg(new RobPtr) 111 val need_gpa_vpn = Reg(UInt(vpnLen.W)) 112 val resp_gpa_gvpn = Reg(UInt(ptePPNLen.W)) 113 val resp_gpa_refill = RegInit(false.B) 114 val resp_s1_level = RegInit(0.U(log2Up(Level + 1).W)) 115 val resp_s1_isLeaf = RegInit(false.B) 116 val resp_s1_isFakePte = RegInit(false.B) 117 val hasGpf = Wire(Vec(Width, Bool())) 118 119 val Sv39Enable = csr.satp.mode === 8.U 120 val Sv48Enable = csr.satp.mode === 9.U 121 val Sv39x4Enable = csr.vsatp.mode === 8.U || csr.hgatp.mode === 8.U 122 val Sv48x4Enable = csr.vsatp.mode === 9.U || csr.hgatp.mode === 9.U 123 val vmEnable = (0 until Width).map(i => !(isHyperInst(i) || virt_out(i)) && ( 124 if (EnbaleTlbDebug) (Sv39Enable || Sv48Enable) 125 else (Sv39Enable || Sv48Enable) && (mode(i) < ModeM)) 126 ) 127 val s2xlateEnable = (0 until Width).map(i => (isHyperInst(i) || virt_out(i)) && (Sv39x4Enable || Sv48x4Enable) && (mode(i) < ModeM)) 128 val portTranslateEnable = (0 until Width).map(i => (vmEnable(i) || s2xlateEnable(i)) && RegEnable(!req(i).bits.no_translate, req(i).valid)) 129 130 // pre fault: check fault before real do translate 131 val prepf = WireInit(VecInit(Seq.fill(Width)(false.B))) 132 val pregpf = WireInit(VecInit(Seq.fill(Width)(false.B))) 133 val preaf = WireInit(VecInit(Seq.fill(Width)(false.B))) 134 val premode = (0 until Width).map(i => Mux(req_in(i).bits.hyperinst, csr.priv.spvp, mode_tmp)) 135 for (i <- 0 until Width) { 136 resp(i).bits.fullva := RegEnable(EffectiveVa(i), req(i).valid) 137 } 138 val prevmEnable = (0 until Width).map(i => !(virt_in || req_in(i).bits.hyperinst) && ( 139 if (EnbaleTlbDebug) (Sv39Enable || Sv48Enable) 140 else (Sv39Enable || Sv48Enable) && (premode(i) < ModeM)) 141 ) 142 val pres2xlateEnable = (0 until Width).map(i => (virt_in || req_in(i).bits.hyperinst) && (Sv39x4Enable || Sv48x4Enable) && (premode(i) < ModeM)) 143 144 (0 until Width).foreach{i => 145 146 val pmm = WireInit(0.U(2.W)) 147 148 when (ifetch || req(i).bits.hlvx) { 149 pmm := 0.U 150 } .elsewhen (premode(i) === ModeM) { 151 pmm := csr.pmm.mseccfg 152 } .elsewhen (!(virt_in || req_in(i).bits.hyperinst) && premode(i) === ModeS) { 153 pmm := csr.pmm.menvcfg 154 } .elsewhen ((virt_in || req_in(i).bits.hyperinst) && premode(i) === ModeS) { 155 pmm := csr.pmm.henvcfg 156 } .elsewhen (req_in(i).bits.hyperinst && csr.priv.imode === ModeU) { 157 pmm := csr.pmm.hstatus 158 } .elsewhen (premode(i) === ModeU) { 159 pmm := csr.pmm.senvcfg 160 } 161 162 when (prevmEnable(i) || (pres2xlateEnable(i) && csr.vsatp.mode =/= 0.U)) { 163 when (pmm === PMLEN7) { 164 EffectiveVa(i) := SignExt(req_in(i).bits.fullva(56, 0), XLEN) 165 } .elsewhen (pmm === PMLEN16) { 166 EffectiveVa(i) := SignExt(req_in(i).bits.fullva(47, 0), XLEN) 167 } .otherwise { 168 EffectiveVa(i) := req_in(i).bits.fullva 169 } 170 } .otherwise { 171 when (pmm === PMLEN7) { 172 EffectiveVa(i) := ZeroExt(req_in(i).bits.fullva(56, 0), XLEN) 173 } .elsewhen (pmm === PMLEN16) { 174 EffectiveVa(i) := ZeroExt(req_in(i).bits.fullva(47, 0), XLEN) 175 } .otherwise { 176 EffectiveVa(i) := req_in(i).bits.fullva 177 } 178 } 179 180 val pf48 = SignExt(EffectiveVa(i)(47, 0), XLEN) =/= EffectiveVa(i) 181 val pf39 = SignExt(EffectiveVa(i)(38, 0), XLEN) =/= EffectiveVa(i) 182 val gpf48 = EffectiveVa(i)(XLEN - 1, 48 + 2) =/= 0.U 183 val gpf39 = EffectiveVa(i)(XLEN - 1, 39 + 2) =/= 0.U 184 val af = EffectiveVa(i)(XLEN - 1, PAddrBits) =/= 0.U 185 when (req(i).valid && req(i).bits.checkfullva) { 186 when (prevmEnable(i) || pres2xlateEnable(i)) { 187 when (req_in_s2xlate(i) === onlyStage2) { 188 when (Sv48x4Enable) { 189 pregpf(i) := gpf48 190 } .elsewhen (Sv39x4Enable) { 191 pregpf(i) := gpf39 192 } 193 } .otherwise { 194 when (Sv48Enable) { 195 prepf(i) := pf48 196 } .elsewhen (Sv39Enable) { 197 prepf(i) := pf39 198 } 199 } 200 } .otherwise { 201 preaf(i) := af 202 } 203 } 204 } 205 206 val refill = ptw.resp.fire && !(ptw.resp.bits.getGpa) && !need_gpa && !need_gpa_wire && !flush_mmu 207 // prevent ptw refill when: 1) it's a getGpa request; 2) l1tlb is in need_gpa state; 3) mmu is being flushed. 208 209 refill_to_mem := DontCare 210 val entries = Module(new TlbStorageWrapper(Width, q, nRespDups)) 211 entries.io.base_connect(sfence, csr, csr.satp) 212 if (q.outReplace) { io.replace <> entries.io.replace } 213 for (i <- 0 until Width) { 214 entries.io.r_req_apply(io.requestor(i).req.valid, get_pn(req_in(i).bits.vaddr), i, req_in_s2xlate(i)) 215 entries.io.w_apply(refill, ptw.resp.bits) 216 // TODO: RegNext enable:req.valid 217 resp(i).bits.debug.isFirstIssue := RegEnable(req(i).bits.debug.isFirstIssue, req(i).valid) 218 resp(i).bits.debug.robIdx := RegEnable(req(i).bits.debug.robIdx, req(i).valid) 219 } 220 221 // read TLB, get hit/miss, paddr, perm bits 222 val readResult = (0 until Width).map(TLBRead(_)) 223 val hitVec = readResult.map(_._1) 224 val missVec = readResult.map(_._2) 225 val pmp_addr = readResult.map(_._3) 226 val perm = readResult.map(_._4) 227 val g_perm = readResult.map(_._5) 228 val pbmt = readResult.map(_._6) 229 val g_pbmt = readResult.map(_._7) 230 // check pmp use paddr (for timing optization, use pmp_addr here) 231 // check permisson 232 (0 until Width).foreach{i => 233 val noTranslateReg = RegNext(req(i).bits.no_translate) 234 val addr = Mux(noTranslateReg, req(i).bits.pmp_addr, pmp_addr(i)) 235 pmp_check(addr, req_out(i).size, req_out(i).cmd, noTranslateReg, i) 236 for (d <- 0 until nRespDups) { 237 pbmt_check(i, d, pbmt(i)(d), g_pbmt(i)(d), req_out_s2xlate(i)) 238 perm_check(perm(i)(d), req_out(i).cmd, i, d, g_perm(i)(d), req_out(i).hlvx, req_out_s2xlate(i), prepf(i), pregpf(i), preaf(i)) 239 } 240 hasGpf(i) := hitVec(i) && (resp(i).bits.excp(0).gpf.ld || resp(i).bits.excp(0).gpf.st || resp(i).bits.excp(0).gpf.instr) 241 } 242 243 // handle block or non-block io 244 // for non-block io, just return the above result, send miss to ptw 245 // for block io, hold the request, send miss to ptw, 246 // when ptw back, return the result 247 (0 until Width) foreach {i => 248 if (Block(i)) handle_block(i) 249 else handle_nonblock(i) 250 } 251 io.ptw.resp.ready := true.B 252 253 /************************ main body above | method/log/perf below ****************************/ 254 def TLBRead(i: Int) = { 255 val (e_hit, e_ppn, e_perm, e_g_perm, e_s2xlate, e_pbmt, e_g_pbmt) = entries.io.r_resp_apply(i) 256 val (p_hit, p_ppn, p_pbmt, p_perm, p_gvpn, p_g_pbmt, p_g_perm, p_s2xlate, p_s1_level, p_s1_isLeaf, p_s1_isFakePte) = ptw_resp_bypass(get_pn(req_in(i).bits.vaddr), req_in_s2xlate(i)) 257 val enable = portTranslateEnable(i) 258 val isOnlys2xlate = req_out_s2xlate(i) === onlyStage2 259 val need_gpa_vpn_hit = need_gpa_vpn === get_pn(req_out(i).vaddr) 260 val isitlb = TlbCmd.isExec(req_out(i).cmd) 261 val isPrefetch = req_out(i).isPrefetch 262 val currentRedirect = req_out(i).debug.robIdx.needFlush(redirect) 263 val lastCycleRedirect = req_out(i).debug.robIdx.needFlush(RegNext(redirect)) 264 265 when (!isitlb && need_gpa_robidx.needFlush(redirect) || isitlb && flush_pipe(i)){ 266 need_gpa := false.B 267 resp_gpa_refill := false.B 268 need_gpa_vpn := 0.U 269 }.elsewhen (req_out_v(i) && !p_hit && !(resp_gpa_refill && need_gpa_vpn_hit) && !isOnlys2xlate && hasGpf(i) && need_gpa === false.B && !io.requestor(i).req_kill && !isPrefetch && !currentRedirect && !lastCycleRedirect) { 270 need_gpa_wire := true.B 271 need_gpa := true.B 272 need_gpa_vpn := get_pn(req_out(i).vaddr) 273 resp_gpa_refill := false.B 274 need_gpa_robidx := req_out(i).debug.robIdx 275 }.elsewhen (ptw.resp.fire && need_gpa && need_gpa_vpn === ptw.resp.bits.getVpn(need_gpa_vpn)) { 276 resp_gpa_gvpn := Mux(ptw.resp.bits.s2xlate === onlyStage2, ptw.resp.bits.s2.entry.tag, ptw.resp.bits.s1.genGVPN(need_gpa_vpn)) 277 resp_s1_level := ptw.resp.bits.s1.entry.level.get 278 resp_s1_isLeaf := ptw.resp.bits.s1.isLeaf() 279 resp_s1_isFakePte := ptw.resp.bits.s1.isFakePte() 280 resp_gpa_refill := true.B 281 } 282 283 when (req_out_v(i) && hasGpf(i) && resp_gpa_refill && need_gpa_vpn_hit){ 284 need_gpa := false.B 285 } 286 287 val hit = e_hit || p_hit 288 val miss = (!hit && enable) || hasGpf(i) && !p_hit && !(resp_gpa_refill && need_gpa_vpn_hit) && !isOnlys2xlate && !isPrefetch && !lastCycleRedirect 289 hit.suggestName(s"hit_read_${i}") 290 miss.suggestName(s"miss_read_${i}") 291 292 val vaddr = SignExt(req_out(i).vaddr, PAddrBits) 293 resp(i).bits.miss := miss 294 resp(i).bits.ptwBack := ptw.resp.fire 295 resp(i).bits.memidx := RegEnable(req_in(i).bits.memidx, req_in(i).valid) 296 resp(i).bits.fastMiss := !hit && enable 297 298 val ppn = WireInit(VecInit(Seq.fill(nRespDups)(0.U(ppnLen.W)))) 299 val pbmt = WireInit(VecInit(Seq.fill(nRespDups)(0.U(ptePbmtLen.W)))) 300 val perm = WireInit(VecInit(Seq.fill(nRespDups)(0.U.asTypeOf(new TlbPermBundle)))) 301 val gvpn = WireInit(VecInit(Seq.fill(nRespDups)(0.U(ptePPNLen.W)))) 302 val level = WireInit(VecInit(Seq.fill(nRespDups)(0.U(log2Up(Level + 1).W)))) 303 val isLeaf = WireInit(VecInit(Seq.fill(nRespDups)(false.B))) 304 val isFakePte = WireInit(VecInit(Seq.fill(nRespDups)(false.B))) 305 val g_pbmt = WireInit(VecInit(Seq.fill(nRespDups)(0.U(ptePbmtLen.W)))) 306 val g_perm = WireInit(VecInit(Seq.fill(nRespDups)(0.U.asTypeOf(new TlbPermBundle)))) 307 val r_s2xlate = WireInit(VecInit(Seq.fill(nRespDups)(0.U(2.W)))) 308 for (d <- 0 until nRespDups) { 309 ppn(d) := Mux(p_hit, p_ppn, e_ppn(d)) 310 pbmt(d) := Mux(p_hit, p_pbmt, e_pbmt(d)) 311 perm(d) := Mux(p_hit, p_perm, e_perm(d)) 312 gvpn(d) := Mux(p_hit, p_gvpn, resp_gpa_gvpn) 313 level(d) := Mux(p_hit, p_s1_level, resp_s1_level) 314 isLeaf(d) := Mux(p_hit, p_s1_isLeaf, resp_s1_isLeaf) 315 isFakePte(d) := Mux(p_hit, p_s1_isFakePte, resp_s1_isFakePte) 316 g_pbmt(d) := Mux(p_hit, p_g_pbmt, e_g_pbmt(d)) 317 g_perm(d) := Mux(p_hit, p_g_perm, e_g_perm(d)) 318 r_s2xlate(d) := Mux(p_hit, p_s2xlate, e_s2xlate(d)) 319 val paddr = Cat(ppn(d), get_off(req_out(i).vaddr)) 320 val vpn_idx = Mux1H(Seq( 321 (isFakePte(d) && csr.vsatp.mode === Sv39) -> 2.U, 322 (isFakePte(d) && csr.vsatp.mode === Sv48) -> 3.U, 323 (!isFakePte(d)) -> (level(d) - 1.U), 324 )) 325 // We use `fullva` here when `isLeaf`, in order to cope with the situation of an unaligned load/store cross page 326 // for example, a `ld` instruction on address 0x81000ffb will be splited into two loads 327 // 1. ld 0x81000ff8. vaddr = 0x81000ff8, fullva = 0x80000ffb 328 // 2. ld 0x81001000. vaddr = 0x81001000, fullva = 0x80000ffb 329 // When load 1 trigger a guest page fault, we should use offset of fullva when generate gpaddr 330 // and when load 2 trigger a guest page fault, we should just use offset of vaddr(all zero). 331 // Also, when onlyS2, if crosspage, gpaddr = vaddr(start address of a new page), else gpaddr = fullva(original vaddr) 332 // By the way, frontend handles the cross page instruction fetch by itself, so TLB doesn't need to do anything extra. 333 // Also, the fullva of iTLB is not used and always zero. crossPageVaddr should never use fullva in iTLB. 334 val crossPageVaddr = Mux(isitlb || req_out(i).fullva(12) =/= vaddr(12), vaddr, req_out(i).fullva) 335 val gpaddr_offset = Mux(isLeaf(d), get_off(crossPageVaddr), Cat(getVpnn(get_pn(crossPageVaddr), vpn_idx), 0.U(log2Up(XLEN/8).W))) 336 val gpaddr = Cat(gvpn(d), gpaddr_offset) 337 resp(i).bits.paddr(d) := Mux(enable, paddr, vaddr) 338 resp(i).bits.gpaddr(d) := Mux(r_s2xlate(d) === onlyStage2, crossPageVaddr, gpaddr) 339 } 340 341 XSDebug(req_out_v(i), p"(${i.U}) hit:${hit} miss:${miss} ppn:${Hexadecimal(ppn(0))} perm:${perm(0)}\n") 342 343 val pmp_paddr = resp(i).bits.paddr(0) 344 345 (hit, miss, pmp_paddr, perm, g_perm, pbmt, g_pbmt) 346 } 347 348 def getVpnn(vpn: UInt, idx: UInt): UInt = { 349 MuxLookup(idx, 0.U)(Seq( 350 0.U -> vpn(vpnnLen - 1, 0), 351 1.U -> vpn(vpnnLen * 2 - 1, vpnnLen), 352 2.U -> vpn(vpnnLen * 3 - 1, vpnnLen * 2), 353 3.U -> vpn(vpnnLen * 4 - 1, vpnnLen * 3)) 354 ) 355 } 356 357 def pmp_check(addr: UInt, size: UInt, cmd: UInt, noTranslate: Bool, idx: Int): Unit = { 358 pmp(idx).valid := resp(idx).valid || noTranslate 359 pmp(idx).bits.addr := addr 360 pmp(idx).bits.size := size 361 pmp(idx).bits.cmd := cmd 362 } 363 364 def pbmt_check(idx: Int, d: Int, pbmt: UInt, g_pbmt: UInt, s2xlate: UInt):Unit = { 365 val onlyS1 = s2xlate === onlyStage1 || s2xlate === noS2xlate 366 val pbmtRes = pbmt 367 val gpbmtRes = g_pbmt 368 val res = MuxLookup(s2xlate, 0.U)(Seq( 369 onlyStage1 -> pbmtRes, 370 onlyStage2 -> gpbmtRes, 371 allStage -> Mux(pbmtRes =/= 0.U, pbmtRes, gpbmtRes), 372 noS2xlate -> pbmtRes 373 )) 374 resp(idx).bits.pbmt(d) := Mux(portTranslateEnable(idx), res, 0.U) 375 } 376 377 // for timing optimization, pmp check is divided into dynamic and static 378 def perm_check(perm: TlbPermBundle, cmd: UInt, idx: Int, nDups: Int, g_perm: TlbPermBundle, hlvx: Bool, s2xlate: UInt, prepf: Bool = false.B, pregpf: Bool = false.B, preaf: Bool = false.B) = { 379 // dynamic: superpage (or full-connected reg entries) -> check pmp when translation done 380 // static: 4K pages (or sram entries) -> check pmp with pre-checked results 381 val hasS2xlate = s2xlate =/= noS2xlate 382 val onlyS1 = s2xlate === onlyStage1 383 val onlyS2 = s2xlate === onlyStage2 384 val allS2xlate = s2xlate === allStage 385 // noS2xlate || onlyS1 -> perm.af 386 // onlyS2 -> g_perm.af 387 // allS2xlate -> perm.af || g_perm.af 388 val af = (!onlyS2 && perm.af) || ((onlyS2 || allS2xlate) && g_perm.af) 389 390 // Stage 1 perm check 391 val pf = perm.pf 392 val isLd = TlbCmd.isRead(cmd) && !TlbCmd.isAmo(cmd) 393 val isSt = TlbCmd.isWrite(cmd) || TlbCmd.isAmo(cmd) 394 val isInst = TlbCmd.isExec(cmd) 395 val ldUpdate = !perm.a && isLd // update A/D through exception 396 val stUpdate = (!perm.a || !perm.d) && isSt // update A/D through exception 397 val instrUpdate = !perm.a && isInst // update A/D through exception 398 val modeCheck = !(mode(idx) === ModeU && !perm.u || mode(idx) === ModeS && perm.u && (!sum(idx) || ifetch)) 399 val ldPermFail = !(modeCheck && Mux(hlvx, perm.x, perm.r || mxr(idx) && perm.x)) 400 val stPermFail = !(modeCheck && perm.w) 401 val instrPermFail = !(modeCheck && perm.x) 402 val ldPf = (ldPermFail || pf) && isLd 403 val stPf = (stPermFail || pf) && isSt 404 val instrPf = (instrPermFail || pf) && isInst 405 val isFakePte = !perm.v && !perm.pf && !perm.af && !onlyS2 406 val isNonLeaf = !(perm.r || perm.w || perm.x) && perm.v && !perm.pf && !perm.af 407 val s1_valid = portTranslateEnable(idx) && !onlyS2 408 409 // Stage 2 perm check 410 val gpf = g_perm.pf 411 val g_ldUpdate = !g_perm.a && isLd 412 val g_stUpdate = (!g_perm.a || !g_perm.d) && isSt 413 val g_instrUpdate = !g_perm.a && isInst 414 val g_ldPermFail = !Mux(hlvx, g_perm.x, (g_perm.r || csr.priv.mxr && g_perm.x)) 415 val g_stPermFail = !g_perm.w 416 val g_instrPermFail = !g_perm.x 417 val ldGpf = (g_ldPermFail || gpf) && isLd 418 val stGpf = (g_stPermFail || gpf) && isSt 419 val instrGpf = (g_instrPermFail || gpf) && isInst 420 val s2_valid = portTranslateEnable(idx) && (onlyS2 || allS2xlate) 421 422 val fault_valid = s1_valid || s2_valid 423 424 // when pf and gpf can't happens simultaneously 425 val hasPf = (ldPf || ldUpdate || stPf || stUpdate || instrPf || instrUpdate) && s1_valid && !af && !isFakePte && !isNonLeaf 426 // Only lsu need check related to high address truncation 427 when (RegNext(prepf || pregpf || preaf)) { 428 resp(idx).bits.isForVSnonLeafPTE := false.B 429 resp(idx).bits.excp(nDups).pf.ld := RegNext(prepf) && isLd 430 resp(idx).bits.excp(nDups).pf.st := RegNext(prepf) && isSt 431 resp(idx).bits.excp(nDups).pf.instr := false.B 432 433 resp(idx).bits.excp(nDups).gpf.ld := RegNext(pregpf) && isLd 434 resp(idx).bits.excp(nDups).gpf.st := RegNext(pregpf) && isSt 435 resp(idx).bits.excp(nDups).gpf.instr := false.B 436 437 resp(idx).bits.excp(nDups).af.ld := RegNext(preaf) && TlbCmd.isRead(cmd) 438 resp(idx).bits.excp(nDups).af.st := RegNext(preaf) && TlbCmd.isWrite(cmd) 439 resp(idx).bits.excp(nDups).af.instr := false.B 440 441 resp(idx).bits.excp(nDups).vaNeedExt := false.B 442 // overwrite miss & gpaddr when exception related to high address truncation happens 443 resp(idx).bits.miss := false.B 444 resp(idx).bits.gpaddr(nDups) := req_out(idx).fullva 445 } .otherwise { 446 // isForVSnonLeafPTE is used only when gpf happens and it caused by a G-stage translation which supports VS-stage translation 447 // it will be sent to CSR in order to modify the m/htinst. 448 // Ref: The RISC-V Instruction Set Manual: Volume II: Privileged Architecture - 19.6.3. Transformed Instruction or Pseudoinstruction for mtinst or htinst 449 val isForVSnonLeafPTE = isNonLeaf || isFakePte 450 resp(idx).bits.isForVSnonLeafPTE := isForVSnonLeafPTE 451 resp(idx).bits.excp(nDups).pf.ld := (ldPf || ldUpdate) && s1_valid && !af && !isFakePte && !isNonLeaf 452 resp(idx).bits.excp(nDups).pf.st := (stPf || stUpdate) && s1_valid && !af && !isFakePte && !isNonLeaf 453 resp(idx).bits.excp(nDups).pf.instr := (instrPf || instrUpdate) && s1_valid && !af && !isFakePte && !isNonLeaf 454 // NOTE: pf need && with !af, page fault has higher priority than access fault 455 // but ptw may also have access fault, then af happens, the translation is wrong. 456 // In this case, pf has lower priority than af 457 458 resp(idx).bits.excp(nDups).gpf.ld := (ldGpf || g_ldUpdate) && s2_valid && !af && !hasPf 459 resp(idx).bits.excp(nDups).gpf.st := (stGpf || g_stUpdate) && s2_valid && !af && !hasPf 460 resp(idx).bits.excp(nDups).gpf.instr := (instrGpf || g_instrUpdate) && s2_valid && !af && !hasPf 461 462 resp(idx).bits.excp(nDups).af.ld := af && TlbCmd.isRead(cmd) && fault_valid 463 resp(idx).bits.excp(nDups).af.st := af && TlbCmd.isWrite(cmd) && fault_valid 464 resp(idx).bits.excp(nDups).af.instr := af && TlbCmd.isExec(cmd) && fault_valid 465 466 resp(idx).bits.excp(nDups).vaNeedExt := true.B 467 } 468 469 resp(idx).bits.excp(nDups).isHyper := isHyperInst(idx) 470 } 471 472 def handle_nonblock(idx: Int): Unit = { 473 io.requestor(idx).resp.valid := req_out_v(idx) 474 io.requestor(idx).req.ready := io.requestor(idx).resp.ready // should always be true 475 XSError(!io.requestor(idx).resp.ready, s"${q.name} port ${idx} is non-block, resp.ready must be true.B") 476 477 val req_need_gpa = hasGpf(idx) 478 val req_s2xlate = Wire(UInt(2.W)) 479 req_s2xlate := MuxCase(noS2xlate, Seq( 480 (!(virt_out(idx) || req_out(idx).hyperinst)) -> noS2xlate, 481 (csr.vsatp.mode =/= 0.U && csr.hgatp.mode =/= 0.U) -> allStage, 482 (csr.vsatp.mode === 0.U) -> onlyStage2, 483 (csr.hgatp.mode === 0.U || req_need_gpa) -> onlyStage1 484 )) 485 486 val ptw_just_back = ptw.resp.fire && req_s2xlate === ptw.resp.bits.s2xlate && ptw.resp.bits.hit(get_pn(req_out(idx).vaddr), csr.satp.asid, csr.vsatp.asid, csr.hgatp.vmid, true, false) 487 // TODO: RegNext enable: ptw.resp.valid ? req.valid 488 val ptw_resp_bits_reg = RegEnable(ptw.resp.bits, ptw.resp.valid) 489 val ptw_already_back = GatedValidRegNext(ptw.resp.fire) && req_s2xlate === ptw_resp_bits_reg.s2xlate && ptw_resp_bits_reg.hit(get_pn(req_out(idx).vaddr), csr.satp.asid, csr.vsatp.asid, csr.hgatp.vmid, allType = true) 490 val ptw_getGpa = req_need_gpa && hitVec(idx) 491 val need_gpa_vpn_hit = need_gpa_vpn === get_pn(req_out(idx).vaddr) 492 493 io.ptw.req(idx).valid := false.B; 494 io.tlbreplay(idx) := false.B; 495 496 when (req_out_v(idx) && missVec(idx)) { 497 // NOTE: for an miss tlb request: either send a ptw request, or ask for a replay 498 when (ptw_just_back || ptw_already_back) { 499 io.tlbreplay(idx) := true.B; 500 } .elsewhen (need_gpa && !need_gpa_vpn_hit && !resp_gpa_refill) { 501 // not send any unrelated ptw request when l1tlb is in need_gpa state 502 io.tlbreplay(idx) := true.B; 503 } .otherwise { 504 io.ptw.req(idx).valid := true.B; 505 } 506 } 507 508 when (io.requestor(idx).req_kill && GatedValidRegNext(io.requestor(idx).req.fire)) { 509 io.ptw.req(idx).valid := false.B 510 io.tlbreplay(idx) := true.B 511 } 512 513 io.ptw.req(idx).bits.vpn := get_pn(req_out(idx).vaddr) 514 io.ptw.req(idx).bits.s2xlate := req_s2xlate 515 io.ptw.req(idx).bits.getGpa := ptw_getGpa 516 io.ptw.req(idx).bits.memidx := req_out(idx).memidx 517 } 518 519 def handle_block(idx: Int): Unit = { 520 // three valid: 1.if exist a entry; 2.if sent to ptw; 3.unset resp.valid 521 io.requestor(idx).req.ready := !req_out_v(idx) || io.requestor(idx).resp.fire 522 // req_out_v for if there is a request, may long latency, fixme 523 524 // miss request entries 525 val req_need_gpa = hasGpf(idx) 526 val miss_req_vpn = get_pn(req_out(idx).vaddr) 527 val miss_req_memidx = req_out(idx).memidx 528 val miss_req_s2xlate = Wire(UInt(2.W)) 529 miss_req_s2xlate := MuxCase(noS2xlate, Seq( 530 (!(virt_out(idx) || req_out(idx).hyperinst)) -> noS2xlate, 531 (csr.vsatp.mode =/= 0.U && csr.hgatp.mode =/= 0.U) -> allStage, 532 (csr.vsatp.mode === 0.U) -> onlyStage2, 533 (csr.hgatp.mode === 0.U || req_need_gpa) -> onlyStage1 534 )) 535 val miss_req_s2xlate_reg = RegEnable(miss_req_s2xlate, io.ptw.req(idx).fire) 536 val hasS2xlate = miss_req_s2xlate_reg =/= noS2xlate 537 val onlyS2 = miss_req_s2xlate_reg === onlyStage2 538 val hit_s1 = io.ptw.resp.bits.s1.hit(miss_req_vpn, Mux(hasS2xlate, csr.vsatp.asid, csr.satp.asid), csr.hgatp.vmid, allType = true, false, hasS2xlate) 539 val hit_s2 = io.ptw.resp.bits.s2.hit(miss_req_vpn, csr.hgatp.vmid) 540 val hit = Mux(onlyS2, hit_s2, hit_s1) && io.ptw.resp.valid && miss_req_s2xlate_reg === io.ptw.resp.bits.s2xlate 541 542 val new_coming_valid = WireInit(false.B) 543 new_coming_valid := req_in(idx).fire && !req_in(idx).bits.kill && !flush_pipe(idx) 544 val new_coming = GatedValidRegNext(new_coming_valid) 545 val miss_wire = new_coming && missVec(idx) 546 val miss_v = ValidHoldBypass(miss_wire, resp(idx).fire, flush_pipe(idx)) 547 val miss_req_v = ValidHoldBypass(miss_wire || (miss_v && flush_mmu && !mmu_flush_pipe), 548 io.ptw.req(idx).fire || resp(idx).fire, flush_pipe(idx)) 549 550 // when ptw resp, check if hit, reset miss_v, resp to lsu/ifu 551 resp(idx).valid := req_out_v(idx) && !(miss_v && portTranslateEnable(idx)) 552 when (io.ptw.resp.fire && hit && req_out_v(idx) && portTranslateEnable(idx)) { 553 val stage1 = io.ptw.resp.bits.s1 554 val stage2 = io.ptw.resp.bits.s2 555 val s2xlate = io.ptw.resp.bits.s2xlate 556 resp(idx).valid := true.B 557 resp(idx).bits.miss := false.B 558 val s1_paddr = Cat(stage1.genPPN(get_pn(req_out(idx).vaddr)), get_off(req_out(idx).vaddr)) 559 val s2_paddr = Cat(stage2.genPPNS2(get_pn(req_out(idx).vaddr)), get_off(req_out(idx).vaddr)) 560 for (d <- 0 until nRespDups) { 561 resp(idx).bits.paddr(d) := Mux(s2xlate =/= noS2xlate, s2_paddr, s1_paddr) 562 resp(idx).bits.gpaddr(d) := s1_paddr 563 pbmt_check(idx, d, io.ptw.resp.bits.s1.entry.pbmt, io.ptw.resp.bits.s2.entry.pbmt, s2xlate) 564 perm_check(stage1, req_out(idx).cmd, idx, d, stage2, req_out(idx).hlvx, s2xlate) 565 } 566 pmp_check(resp(idx).bits.paddr(0), req_out(idx).size, req_out(idx).cmd, false.B, idx) 567 568 // NOTE: the unfiltered req would be handled by Repeater 569 } 570 assert(RegNext(!resp(idx).valid || resp(idx).ready, true.B), "when tlb resp valid, ready should be true, must") 571 assert(RegNext(req_out_v(idx) || !(miss_v || miss_req_v), true.B), "when not req_out_v, should not set miss_v/miss_req_v") 572 573 val ptw_req = io.ptw.req(idx) 574 ptw_req.valid := miss_req_v 575 ptw_req.bits.vpn := miss_req_vpn 576 ptw_req.bits.s2xlate := miss_req_s2xlate 577 ptw_req.bits.getGpa := req_need_gpa && hitVec(idx) 578 ptw_req.bits.memidx := miss_req_memidx 579 580 io.tlbreplay(idx) := false.B 581 582 // NOTE: when flush pipe, tlb should abandon last req 583 // however, some outside modules like icache, dont care flushPipe, and still waiting for tlb resp 584 // just resp valid and raise page fault to go through. The pipe(ifu) will abandon it. 585 if (!q.outsideRecvFlush) { 586 when (req_out_v(idx) && flush_pipe(idx) && portTranslateEnable(idx)) { 587 resp(idx).valid := true.B 588 for (d <- 0 until nRespDups) { 589 resp(idx).bits.pbmt(d) := 0.U 590 resp(idx).bits.excp(d).pf.ld := true.B // sfence happened, pf for not to use this addr 591 resp(idx).bits.excp(d).pf.st := true.B 592 resp(idx).bits.excp(d).pf.instr := true.B 593 } 594 } 595 } 596 } 597 598 // when ptw resp, tlb at refill_idx maybe set to miss by force. 599 // Bypass ptw resp to check. 600 def ptw_resp_bypass(vpn: UInt, s2xlate: UInt) = { 601 // TODO: RegNext enable: ptw.resp.valid 602 val hasS2xlate = s2xlate =/= noS2xlate 603 val onlyS2 = s2xlate === onlyStage2 604 val onlyS1 = s2xlate === onlyStage1 605 val s2xlate_hit = s2xlate === ptw.resp.bits.s2xlate 606 val resp_hit = ptw.resp.bits.hit(vpn, csr.satp.asid, csr.vsatp.asid, csr.hgatp.vmid, true, false) 607 val p_hit = GatedValidRegNext(resp_hit && io.ptw.resp.fire && s2xlate_hit) 608 val ppn_s1 = ptw.resp.bits.s1.genPPN(vpn) 609 val gvpn = Mux(onlyS2, vpn, ppn_s1) 610 val ppn_s2 = ptw.resp.bits.s2.genPPNS2(gvpn) 611 val p_ppn = RegEnable(Mux(s2xlate === onlyStage2 || s2xlate === allStage, ppn_s2, ppn_s1), io.ptw.resp.fire) 612 val p_pbmt = RegEnable(ptw.resp.bits.s1.entry.pbmt,io.ptw.resp.fire) 613 val p_perm = RegEnable(ptwresp_to_tlbperm(ptw.resp.bits.s1), io.ptw.resp.fire) 614 val p_gvpn = RegEnable(Mux(onlyS2, ptw.resp.bits.s2.entry.tag, ptw.resp.bits.s1.genGVPN(vpn)), io.ptw.resp.fire) 615 val p_g_pbmt = RegEnable(ptw.resp.bits.s2.entry.pbmt,io.ptw.resp.fire) 616 val p_g_perm = RegEnable(hptwresp_to_tlbperm(ptw.resp.bits.s2), io.ptw.resp.fire) 617 val p_s2xlate = RegEnable(ptw.resp.bits.s2xlate, io.ptw.resp.fire) 618 val p_s1_level = RegEnable(ptw.resp.bits.s1.entry.level.get, io.ptw.resp.fire) 619 val p_s1_isLeaf = RegEnable(ptw.resp.bits.s1.isLeaf(), io.ptw.resp.fire) 620 val p_s1_isFakePte = RegEnable(ptw.resp.bits.s1.isFakePte(), io.ptw.resp.fire) 621 (p_hit, p_ppn, p_pbmt, p_perm, p_gvpn, p_g_pbmt, p_g_perm, p_s2xlate, p_s1_level, p_s1_isLeaf, p_s1_isFakePte) 622 } 623 624 // perf event 625 val result_ok = req_in.map(a => GatedValidRegNext(a.fire)) 626 val perfEvents = 627 Seq( 628 ("access", PopCount((0 until Width).map{i => if (Block(i)) io.requestor(i).req.fire else portTranslateEnable(i) && result_ok(i) })), 629 ("miss ", PopCount((0 until Width).map{i => if (Block(i)) portTranslateEnable(i) && result_ok(i) && missVec(i) else ptw.req(i).fire })), 630 ) 631 generatePerfEvent() 632 633 // perf log 634 for (i <- 0 until Width) { 635 if (Block(i)) { 636 XSPerfAccumulate(s"access${i}",result_ok(i) && portTranslateEnable(i)) 637 XSPerfAccumulate(s"miss${i}", result_ok(i) && missVec(i)) 638 } else { 639 XSPerfAccumulate("first_access" + Integer.toString(i, 10), result_ok(i) && portTranslateEnable(i) && RegEnable(req(i).bits.debug.isFirstIssue, req(i).valid)) 640 XSPerfAccumulate("access" + Integer.toString(i, 10), result_ok(i) && portTranslateEnable(i)) 641 XSPerfAccumulate("first_miss" + Integer.toString(i, 10), result_ok(i) && portTranslateEnable(i) && missVec(i) && RegEnable(req(i).bits.debug.isFirstIssue, req(i).valid)) 642 XSPerfAccumulate("miss" + Integer.toString(i, 10), result_ok(i) && portTranslateEnable(i) && missVec(i)) 643 } 644 } 645 XSPerfAccumulate("ptw_resp_count", ptw.resp.fire) 646 XSPerfAccumulate("ptw_resp_pf_count", ptw.resp.fire && ptw.resp.bits.s1.pf) 647 648 // Log 649 for(i <- 0 until Width) { 650 XSDebug(req(i).valid, p"req(${i.U}): (${req(i).valid} ${req(i).ready}) ${req(i).bits}\n") 651 XSDebug(resp(i).valid, p"resp(${i.U}): (${resp(i).valid} ${resp(i).ready}) ${resp(i).bits}\n") 652 } 653 654 XSDebug(io.sfence.valid, p"Sfence: ${io.sfence}\n") 655 XSDebug(ParallelOR(req_out_v) || ptw.resp.valid, p"vmEnable:${vmEnable} hit:${Binary(VecInit(hitVec).asUInt)} miss:${Binary(VecInit(missVec).asUInt)}\n") 656 for (i <- ptw.req.indices) { 657 XSDebug(ptw.req(i).fire, p"L2TLB req:${ptw.req(i).bits}\n") 658 } 659 XSDebug(ptw.resp.valid, p"L2TLB resp:${ptw.resp.bits} (v:${ptw.resp.valid}r:${ptw.resp.ready}) \n") 660 661 println(s"${q.name}: page: ${q.NWays} ${q.Associative} ${q.Replacer.get}") 662 663 if (env.EnableDifftest) { 664 for (i <- 0 until Width) { 665 val pf = io.requestor(i).resp.bits.excp(0).pf.instr || io.requestor(i).resp.bits.excp(0).pf.st || io.requestor(i).resp.bits.excp(0).pf.ld 666 val gpf = io.requestor(i).resp.bits.excp(0).gpf.instr || io.requestor(i).resp.bits.excp(0).gpf.st || io.requestor(i).resp.bits.excp(0).gpf.ld 667 val af = io.requestor(i).resp.bits.excp(0).af.instr || io.requestor(i).resp.bits.excp(0).af.st || io.requestor(i).resp.bits.excp(0).af.ld 668 val difftest = DifftestModule(new DiffL1TLBEvent) 669 difftest.coreid := io.hartId 670 difftest.valid := RegNext(io.requestor(i).req.fire) && !io.requestor(i).req_kill && io.requestor(i).resp.fire && !io.requestor(i).resp.bits.miss && !pf && !af && !gpf && portTranslateEnable(i) 671 if (!Seq("itlb", "ldtlb", "sttlb").contains(q.name)) { 672 difftest.valid := false.B 673 } 674 difftest.index := TLBDiffId(p(XSCoreParamsKey).HartId).U 675 difftest.vpn := RegEnable(get_pn(req_in(i).bits.vaddr), req_in(i).valid) 676 difftest.ppn := get_pn(io.requestor(i).resp.bits.paddr(0)) 677 difftest.satp := Cat(csr.satp.mode, csr.satp.asid, csr.satp.ppn) 678 difftest.vsatp := Cat(csr.vsatp.mode, csr.vsatp.asid, csr.vsatp.ppn) 679 difftest.hgatp := Cat(csr.hgatp.mode, csr.hgatp.vmid, csr.hgatp.ppn) 680 val req_need_gpa = gpf 681 val req_s2xlate = Wire(UInt(2.W)) 682 req_s2xlate := MuxCase(noS2xlate, Seq( 683 (!RegNext(virt_in || req_in(i).bits.hyperinst)) -> noS2xlate, 684 (csr.vsatp.mode =/= 0.U && csr.hgatp.mode =/= 0.U) -> allStage, 685 (csr.vsatp.mode === 0.U) -> onlyStage2, 686 (csr.hgatp.mode === 0.U || req_need_gpa) -> onlyStage1 687 )) 688 difftest.s2xlate := req_s2xlate 689 } 690 } 691} 692 693object TLBDiffId { 694 var i: Int = 0 695 var lastHartId: Int = -1 696 def apply(hartId: Int): Int = { 697 if (lastHartId != hartId) { 698 i = 0 699 lastHartId = hartId 700 } 701 i += 1 702 i - 1 703 } 704} 705 706class TLBNonBlock(Width: Int, nRespDups: Int = 1, q: TLBParameters)(implicit p: Parameters) extends TLB(Width, nRespDups, Seq.fill(Width)(false), q) 707class TLBBLock(Width: Int, nRespDups: Int = 1, q: TLBParameters)(implicit p: Parameters) extends TLB(Width, nRespDups, Seq.fill(Width)(true), q) 708 709class TlbReplace(Width: Int, q: TLBParameters)(implicit p: Parameters) extends TlbModule { 710 val io = IO(new TlbReplaceIO(Width, q)) 711 712 if (q.Associative == "fa") { 713 val re = ReplacementPolicy.fromString(q.Replacer, q.NWays) 714 re.access(io.page.access.map(_.touch_ways)) 715 io.page.refillIdx := re.way 716 } else { // set-acco && plru 717 val re = ReplacementPolicy.fromString(q.Replacer, q.NSets, q.NWays) 718 re.access(io.page.access.map(_.sets), io.page.access.map(_.touch_ways)) 719 io.page.refillIdx := { if (q.NWays == 1) 0.U else re.way(io.page.chosen_set) } 720 } 721} 722