1/*************************************************************************************** 2* Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences 3* Copyright (c) 2020-2021 Peng Cheng Laboratory 4* 5* XiangShan is licensed under Mulan PSL v2. 6* You can use this software according to the terms and conditions of the Mulan PSL v2. 7* You may obtain a copy of Mulan PSL v2 at: 8* http://license.coscl.org.cn/MulanPSL2 9 10* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, 11* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, 12* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. 13* 14* See the Mulan PSL v2 for more details. 15***************************************************************************************/ 16 17package xiangshan.cache.mmu 18 19import org.chipsalliance.cde.config.Parameters 20import chisel3._ 21import chisel3.util._ 22import difftest._ 23import freechips.rocketchip.util.SRAMAnnotation 24import xiangshan._ 25import utils._ 26import utility._ 27import xiangshan.backend.fu.{PMPChecker, PMPReqBundle, PMPConfig => XSPMPConfig} 28import xiangshan.backend.rob.RobPtr 29import xiangshan.backend.fu.util.HasCSRConst 30import freechips.rocketchip.rocket.PMPConfig 31 32/** TLB module 33 * support block request and non-block request io at the same time 34 * return paddr at next cycle, then go for pmp/pma check 35 * @param Width: The number of requestors 36 * @param Block: Blocked or not for each requestor ports 37 * @param q: TLB Parameters, like entry number, each TLB has its own parameters 38 * @param p: XiangShan Paramemters, like XLEN 39 */ 40 41class TLB(Width: Int, nRespDups: Int = 1, Block: Seq[Boolean], q: TLBParameters)(implicit p: Parameters) extends TlbModule 42 with HasCSRConst 43 with HasPerfEvents 44{ 45 val io = IO(new TlbIO(Width, nRespDups, q)) 46 47 val req = io.requestor.map(_.req) 48 val resp = io.requestor.map(_.resp) 49 val ptw = io.ptw 50 val pmp = io.pmp 51 val refill_to_mem = io.refill_to_mem 52 53 /** Sfence.vma & Svinval 54 * Sfence.vma will 1. flush old entries 2. flush inflight 3. flush pipe 55 * Svinval will 1. flush old entries 2. flush inflight 56 * So, Svinval will not flush pipe, which means 57 * it should not drop reqs from pipe and should return right resp 58 */ 59 val sfence = DelayN(io.sfence, q.fenceDelay) 60 val csr = io.csr 61 val satp = DelayN(io.csr.satp, q.fenceDelay) 62 val vsatp = DelayN(io.csr.vsatp, q.fenceDelay) 63 val hgatp = DelayN(io.csr.hgatp, q.fenceDelay) 64 65 val flush_mmu = DelayN(sfence.valid || csr.satp.changed || csr.vsatp.changed || csr.hgatp.changed, q.fenceDelay) 66 val mmu_flush_pipe = DelayN(sfence.valid && sfence.bits.flushPipe, q.fenceDelay) // for svinval, won't flush pipe 67 val flush_pipe = io.flushPipe 68 val redirect = io.redirect 69 val req_in = req 70 val req_out = req.map(a => RegEnable(a.bits, a.fire)) 71 val req_out_v = (0 until Width).map(i => ValidHold(req_in(i).fire && !req_in(i).bits.kill, resp(i).fire, flush_pipe(i))) 72 73 val isHyperInst = (0 until Width).map(i => req_out_v(i) && req_out(i).hyperinst) 74 75 // ATTENTION: csr and flush from backend are delayed. csr should not be later than flush. 76 // because, csr will influence tlb behavior. 77 val ifecth = if (q.fetchi) true.B else false.B 78 val mode_tmp = if (q.useDmode) csr.priv.dmode else csr.priv.imode 79 val mode = (0 until Width).map(i => Mux(isHyperInst(i), csr.priv.spvp, mode_tmp)) 80 val virt_in = csr.priv.virt 81 val virt_out = req.map(a => RegEnable(csr.priv.virt, a.fire)) 82 val sum = (0 until Width).map(i => Mux(virt_out(i) || isHyperInst(i), io.csr.priv.vsum, io.csr.priv.sum)) 83 val mxr = (0 until Width).map(i => Mux(virt_out(i) || isHyperInst(i), io.csr.priv.vmxr || io.csr.priv.mxr, io.csr.priv.mxr)) 84 val req_in_s2xlate = (0 until Width).map(i => MuxCase(noS2xlate, Seq( 85 (!(virt_in || req_in(i).bits.hyperinst)) -> noS2xlate, 86 (csr.vsatp.mode =/= 0.U && csr.hgatp.mode =/= 0.U) -> allStage, 87 (csr.vsatp.mode === 0.U) -> onlyStage2, 88 (csr.hgatp.mode === 0.U) -> onlyStage1 89 ))) 90 val req_out_s2xlate = (0 until Width).map(i => MuxCase(noS2xlate, Seq( 91 (!(virt_out(i) || isHyperInst(i))) -> noS2xlate, 92 (csr.vsatp.mode =/= 0.U && csr.hgatp.mode =/= 0.U) -> allStage, 93 (csr.vsatp.mode === 0.U) -> onlyStage2, 94 (csr.hgatp.mode === 0.U) -> onlyStage1 95 ))) 96 val need_gpa = RegInit(false.B) 97 val need_gpa_robidx = Reg(new RobPtr) 98 val need_gpa_vpn = Reg(UInt(vpnLen.W)) 99 val resp_gpa_gvpn = Reg(UInt(ptePPNLen.W)) 100 val resp_gpa_refill = RegInit(false.B) 101 val resp_s1_level = RegInit(0.U(log2Up(Level + 1).W)) 102 val resp_s1_isLeaf = RegInit(false.B) 103 val resp_s1_isFakePte = RegInit(false.B) 104 val hasGpf = Wire(Vec(Width, Bool())) 105 106 val Sv39Enable = satp.mode === 8.U 107 val Sv48Enable = satp.mode === 9.U 108 val Sv39x4Enable = vsatp.mode === 8.U || hgatp.mode === 8.U 109 val Sv48x4Enable = vsatp.mode === 9.U || hgatp.mode === 9.U 110 val vmEnable = (0 until Width).map(i => !(isHyperInst(i) || virt_out(i)) && ( 111 if (EnbaleTlbDebug) (Sv39Enable || Sv48Enable) 112 else (Sv39Enable || Sv48Enable) && (mode(i) < ModeM)) 113 ) 114 val s2xlateEnable = (0 until Width).map(i => (isHyperInst(i) || virt_out(i)) && (Sv39x4Enable || Sv48x4Enable) && (mode(i) < ModeM)) 115 val portTranslateEnable = (0 until Width).map(i => (vmEnable(i) || s2xlateEnable(i)) && RegEnable(!req(i).bits.no_translate, req(i).valid)) 116 117 118 val refill = ptw.resp.fire && !(ptw.resp.bits.getGpa) && !flush_mmu 119 refill_to_mem := DontCare 120 val entries = Module(new TlbStorageWrapper(Width, q, nRespDups)) 121 entries.io.base_connect(sfence, csr, satp) 122 if (q.outReplace) { io.replace <> entries.io.replace } 123 for (i <- 0 until Width) { 124 entries.io.r_req_apply(io.requestor(i).req.valid, get_pn(req_in(i).bits.vaddr), i, req_in_s2xlate(i)) 125 entries.io.w_apply(refill, ptw.resp.bits) 126 // TODO: RegNext enable:req.valid 127 resp(i).bits.debug.isFirstIssue := RegEnable(req(i).bits.debug.isFirstIssue, req(i).valid) 128 resp(i).bits.debug.robIdx := RegEnable(req(i).bits.debug.robIdx, req(i).valid) 129 } 130 131 // read TLB, get hit/miss, paddr, perm bits 132 val readResult = (0 until Width).map(TLBRead(_)) 133 val hitVec = readResult.map(_._1) 134 val missVec = readResult.map(_._2) 135 val pmp_addr = readResult.map(_._3) 136 val perm = readResult.map(_._4) 137 val g_perm = readResult.map(_._5) 138 val pbmt = readResult.map(_._6) 139 val g_pbmt = readResult.map(_._7) 140 // check pmp use paddr (for timing optization, use pmp_addr here) 141 // check permisson 142 (0 until Width).foreach{i => 143 when (RegNext(req(i).bits.no_translate)) { 144 pmp_check(req(i).bits.pmp_addr, req_out(i).size, req_out(i).cmd, i) 145 } .otherwise { 146 pmp_check(pmp_addr(i), req_out(i).size, req_out(i).cmd, i) 147 } 148 for (d <- 0 until nRespDups) { 149 pbmt_check(i, d, pbmt(i)(d), g_pbmt(i)(d), req_out_s2xlate(i)) 150 perm_check(perm(i)(d), req_out(i).cmd, i, d, g_perm(i)(d), req_out(i).hlvx, req_out_s2xlate(i)) 151 } 152 hasGpf(i) := resp(i).bits.excp(0).gpf.ld || resp(i).bits.excp(0).gpf.st || resp(i).bits.excp(0).gpf.instr 153 } 154 155 // handle block or non-block io 156 // for non-block io, just return the above result, send miss to ptw 157 // for block io, hold the request, send miss to ptw, 158 // when ptw back, return the result 159 (0 until Width) foreach {i => 160 if (Block(i)) handle_block(i) 161 else handle_nonblock(i) 162 } 163 io.ptw.resp.ready := true.B 164 165 /************************ main body above | method/log/perf below ****************************/ 166 def TLBRead(i: Int) = { 167 val (e_hit, e_ppn, e_perm, e_g_perm, e_s2xlate, e_pbmt, e_g_pbmt) = entries.io.r_resp_apply(i) 168 val (p_hit, p_ppn, p_pbmt, p_perm, p_gvpn, p_g_pbmt, p_g_perm, p_s2xlate, p_s1_level, p_s1_isLeaf, p_s1_isFakePte) = ptw_resp_bypass(get_pn(req_in(i).bits.vaddr), req_in_s2xlate(i)) 169 val enable = portTranslateEnable(i) 170 val isOnlys2xlate = req_out_s2xlate(i) === onlyStage2 171 val need_gpa_vpn_hit = need_gpa_vpn === get_pn(req_out(i).vaddr) 172 val isitlb = TlbCmd.isExec(req_out(i).cmd) 173 174 when (!isitlb && need_gpa_robidx.needFlush(redirect) || isitlb && flush_pipe(i)){ 175 need_gpa := false.B 176 resp_gpa_refill := false.B 177 need_gpa_vpn := 0.U 178 }.elsewhen (req_out_v(i) && !p_hit && !(resp_gpa_refill && need_gpa_vpn_hit) && !isOnlys2xlate && hasGpf(i) && need_gpa === false.B && !io.requestor(i).req_kill) { 179 need_gpa := true.B 180 need_gpa_vpn := get_pn(req_out(i).vaddr) 181 resp_gpa_refill := false.B 182 need_gpa_robidx := req_out(i).debug.robIdx 183 }.elsewhen (ptw.resp.fire && need_gpa && need_gpa_vpn === ptw.resp.bits.getVpn(need_gpa_vpn)) { 184 resp_gpa_gvpn := ptw.resp.bits.s1.genPPN(need_gpa_vpn) 185 resp_s1_level := ptw.resp.bits.s1.entry.level.get 186 resp_s1_isLeaf := ptw.resp.bits.s1.isLeaf() 187 resp_s1_isFakePte := ptw.resp.bits.s1.isFakePte() 188 resp_gpa_refill := true.B 189 } 190 191 when (req_out_v(i) && hasGpf(i) && resp_gpa_refill && need_gpa_vpn_hit ){ 192 need_gpa := false.B 193 } 194 195 TimeOutAssert(need_gpa && !resp_gpa_refill, timeOutThreshold, s"port${i} need gpa long time not refill.") 196 197 val hit = e_hit || p_hit 198 val miss = (!hit && enable) || hasGpf(i) && !p_hit && !(resp_gpa_refill && need_gpa_vpn_hit) && !isOnlys2xlate 199 hit.suggestName(s"hit_read_${i}") 200 miss.suggestName(s"miss_read_${i}") 201 202 val vaddr = SignExt(req_out(i).vaddr, PAddrBits) 203 resp(i).bits.miss := miss 204 resp(i).bits.ptwBack := ptw.resp.fire 205 resp(i).bits.memidx := RegEnable(req_in(i).bits.memidx, req_in(i).valid) 206 207 val ppn = WireInit(VecInit(Seq.fill(nRespDups)(0.U(ppnLen.W)))) 208 val pbmt = WireInit(VecInit(Seq.fill(nRespDups)(0.U(ptePbmtLen.W)))) 209 val perm = WireInit(VecInit(Seq.fill(nRespDups)(0.U.asTypeOf(new TlbPermBundle)))) 210 val gvpn = WireInit(VecInit(Seq.fill(nRespDups)(0.U(vpnLen.W)))) 211 val level = WireInit(VecInit(Seq.fill(nRespDups)(0.U(log2Up(Level + 1).W)))) 212 val isLeaf = WireInit(VecInit(Seq.fill(nRespDups)(false.B))) 213 val isFakePte = WireInit(VecInit(Seq.fill(nRespDups)(false.B))) 214 val g_pbmt = WireInit(VecInit(Seq.fill(nRespDups)(0.U(ptePbmtLen.W)))) 215 val g_perm = WireInit(VecInit(Seq.fill(nRespDups)(0.U.asTypeOf(new TlbPermBundle)))) 216 val r_s2xlate = WireInit(VecInit(Seq.fill(nRespDups)(0.U(2.W)))) 217 for (d <- 0 until nRespDups) { 218 ppn(d) := Mux(p_hit, p_ppn, e_ppn(d)) 219 pbmt(d) := Mux(p_hit, p_pbmt, e_pbmt(d)) 220 perm(d) := Mux(p_hit, p_perm, e_perm(d)) 221 gvpn(d) := Mux(p_hit, p_gvpn, resp_gpa_gvpn) 222 level(d) := Mux(p_hit, p_s1_level, resp_s1_level) 223 isLeaf(d) := Mux(p_hit, p_s1_isLeaf, resp_s1_isLeaf) 224 isFakePte(d) := Mux(p_hit, p_s1_isFakePte, resp_s1_isFakePte) 225 g_pbmt(d) := Mux(p_hit, p_g_pbmt, e_g_pbmt(d)) 226 g_perm(d) := Mux(p_hit, p_g_perm, e_g_perm(d)) 227 r_s2xlate(d) := Mux(p_hit, p_s2xlate, e_s2xlate(d)) 228 val paddr = Cat(ppn(d), get_off(req_out(i).vaddr)) 229 val vpn_idx = Mux1H(Seq( 230 (isFakePte(d) && vsatp.mode === Sv39) -> 2.U, 231 (isFakePte(d) && vsatp.mode === Sv48) -> 3.U, 232 (!isFakePte(d)) -> (level(d) - 1.U), 233 )) 234 val gpaddr_offset = Mux(isLeaf(d), get_off(req_out(i).vaddr), Cat(getVpnn(get_pn(req_out(i).vaddr), vpn_idx), 0.U(log2Up(XLEN/8).W))) 235 val gpaddr = Cat(gvpn(d), gpaddr_offset) 236 resp(i).bits.paddr(d) := Mux(enable, paddr, vaddr) 237 resp(i).bits.gpaddr(d) := Mux(r_s2xlate(d) === onlyStage2, vaddr, gpaddr) 238 } 239 240 XSDebug(req_out_v(i), p"(${i.U}) hit:${hit} miss:${miss} ppn:${Hexadecimal(ppn(0))} perm:${perm(0)}\n") 241 242 val pmp_paddr = resp(i).bits.paddr(0) 243 244 (hit, miss, pmp_paddr, perm, g_perm, pbmt, g_pbmt) 245 } 246 247 def getVpnn(vpn: UInt, idx: UInt): UInt = { 248 MuxLookup(idx, 0.U)(Seq( 249 0.U -> vpn(vpnnLen - 1, 0), 250 1.U -> vpn(vpnnLen * 2 - 1, vpnnLen), 251 2.U -> vpn(vpnnLen * 3 - 1, vpnnLen * 2), 252 3.U -> vpn(vpnnLen * 4 - 1, vpnnLen * 3)) 253 ) 254 } 255 256 def pmp_check(addr: UInt, size: UInt, cmd: UInt, idx: Int): Unit = { 257 pmp(idx).valid := resp(idx).valid 258 pmp(idx).bits.addr := addr 259 pmp(idx).bits.size := size 260 pmp(idx).bits.cmd := cmd 261 } 262 263 def pbmt_check(idx: Int, d: Int, pbmt: UInt, g_pbmt: UInt, s2xlate: UInt):Unit = { 264 val onlyS1 = s2xlate === onlyStage1 || s2xlate === noS2xlate 265 resp(idx).bits.pbmt(d) := Mux( 266 portTranslateEnable(idx), 267 Mux(onlyS1, pbmt, g_pbmt), 268 0.U 269 ) 270 } 271 272 // for timing optimization, pmp check is divided into dynamic and static 273 def perm_check(perm: TlbPermBundle, cmd: UInt, idx: Int, nDups: Int, g_perm: TlbPermBundle, hlvx: Bool, s2xlate: UInt) = { 274 // dynamic: superpage (or full-connected reg entries) -> check pmp when translation done 275 // static: 4K pages (or sram entries) -> check pmp with pre-checked results 276 val hasS2xlate = s2xlate =/= noS2xlate 277 val onlyS1 = s2xlate === onlyStage1 278 val onlyS2 = s2xlate === onlyStage2 279 val af = perm.af || (hasS2xlate && g_perm.af) 280 281 // Stage 1 perm check 282 val pf = perm.pf 283 val ldUpdate = !perm.a && TlbCmd.isRead(cmd) && !TlbCmd.isAmo(cmd) // update A/D through exception 284 val stUpdate = (!perm.a || !perm.d) && (TlbCmd.isWrite(cmd) || TlbCmd.isAmo(cmd)) // update A/D through exception 285 val instrUpdate = !perm.a && TlbCmd.isExec(cmd) // update A/D through exception 286 val modeCheck = !(mode(idx) === ModeU && !perm.u || mode(idx) === ModeS && perm.u && (!sum(idx) || ifecth)) 287 val ldPermFail = !(modeCheck && Mux(hlvx, perm.x, perm.r || mxr(idx) && perm.x)) 288 val stPermFail = !(modeCheck && perm.w) 289 val instrPermFail = !(modeCheck && perm.x) 290 val ldPf = (ldPermFail || pf) && (TlbCmd.isRead(cmd) && !TlbCmd.isAmo(cmd)) 291 val stPf = (stPermFail || pf) && (TlbCmd.isWrite(cmd) || TlbCmd.isAmo(cmd)) 292 val instrPf = (instrPermFail || pf) && TlbCmd.isExec(cmd) 293 val s1_valid = portTranslateEnable(idx) && !onlyS2 294 295 // Stage 2 perm check 296 val gpf = g_perm.pf 297 val g_ldUpdate = !g_perm.a && TlbCmd.isRead(cmd) && !TlbCmd.isAmo(cmd) 298 val g_stUpdate = (!g_perm.a || !g_perm.d) && (TlbCmd.isWrite(cmd) || TlbCmd.isAmo(cmd)) 299 val g_instrUpdate = !g_perm.a && TlbCmd.isExec(cmd) 300 val g_ldPermFail = !Mux(hlvx, g_perm.x, (g_perm.r || io.csr.priv.mxr && g_perm.x)) 301 val g_stPermFail = !g_perm.w 302 val g_instrPermFail = !g_perm.x 303 val ldGpf = (g_ldPermFail || gpf) && (TlbCmd.isRead(cmd) && !TlbCmd.isAmo(cmd)) 304 val stGpf = (g_stPermFail || gpf) && (TlbCmd.isWrite(cmd) || TlbCmd.isAmo(cmd)) 305 val instrGpf = (g_instrPermFail || gpf) && TlbCmd.isExec(cmd) 306 val s2_valid = hasS2xlate && !onlyS1 && portTranslateEnable(idx) 307 308 val fault_valid = s1_valid || s2_valid 309 310 // when pf and gpf can't happens simultaneously 311 val hasPf = (ldPf || ldUpdate || stPf || stUpdate || instrPf || instrUpdate) && s1_valid && !af 312 resp(idx).bits.excp(nDups).pf.ld := (ldPf || ldUpdate) && s1_valid && !af 313 resp(idx).bits.excp(nDups).pf.st := (stPf || stUpdate) && s1_valid && !af 314 resp(idx).bits.excp(nDups).pf.instr := (instrPf || instrUpdate) && s1_valid && !af 315 // NOTE: pf need && with !af, page fault has higher priority than access fault 316 // but ptw may also have access fault, then af happens, the translation is wrong. 317 // In this case, pf has lower priority than af 318 319 resp(idx).bits.excp(nDups).gpf.ld := (ldGpf || g_ldUpdate) && s2_valid && !af && !hasPf 320 resp(idx).bits.excp(nDups).gpf.st := (stGpf || g_stUpdate) && s2_valid && !af && !hasPf 321 resp(idx).bits.excp(nDups).gpf.instr := (instrGpf || g_instrUpdate) && s2_valid && !af && !hasPf 322 323 resp(idx).bits.excp(nDups).af.ld := af && TlbCmd.isRead(cmd) && fault_valid 324 resp(idx).bits.excp(nDups).af.st := af && TlbCmd.isWrite(cmd) && fault_valid 325 resp(idx).bits.excp(nDups).af.instr := af && TlbCmd.isExec(cmd) && fault_valid 326 327 328 } 329 330 def handle_nonblock(idx: Int): Unit = { 331 io.requestor(idx).resp.valid := req_out_v(idx) 332 io.requestor(idx).req.ready := io.requestor(idx).resp.ready // should always be true 333 XSError(!io.requestor(idx).resp.ready, s"${q.name} port ${idx} is non-block, resp.ready must be true.B") 334 335 val req_need_gpa = hasGpf(idx) 336 val req_s2xlate = Wire(UInt(2.W)) 337 req_s2xlate := MuxCase(noS2xlate, Seq( 338 (!(virt_out(idx) || req_out(idx).hyperinst)) -> noS2xlate, 339 (csr.vsatp.mode =/= 0.U && csr.hgatp.mode =/= 0.U) -> allStage, 340 (csr.vsatp.mode === 0.U) -> onlyStage2, 341 (csr.hgatp.mode === 0.U || req_need_gpa) -> onlyStage1 342 )) 343 344 val ptw_just_back = ptw.resp.fire && req_s2xlate === ptw.resp.bits.s2xlate && ptw.resp.bits.hit(get_pn(req_out(idx).vaddr), io.csr.satp.asid, io.csr.vsatp.asid, io.csr.hgatp.vmid, true, false) 345 // TODO: RegNext enable: ptw.resp.valid ? req.valid 346 val ptw_resp_bits_reg = RegEnable(ptw.resp.bits, ptw.resp.valid) 347 val ptw_already_back = GatedValidRegNext(ptw.resp.fire) && req_s2xlate === ptw_resp_bits_reg.s2xlate && ptw_resp_bits_reg.hit(get_pn(req_out(idx).vaddr), io.csr.satp.asid, io.csr.vsatp.asid, io.csr.hgatp.vmid, allType = true) 348 io.ptw.req(idx).valid := req_out_v(idx) && missVec(idx) && !(ptw_just_back || ptw_already_back) // TODO: remove the regnext, timing 349 io.tlbreplay(idx) := req_out_v(idx) && missVec(idx) && (ptw_just_back || ptw_already_back) 350 when (io.requestor(idx).req_kill && GatedValidRegNext(io.requestor(idx).req.fire)) { 351 io.ptw.req(idx).valid := false.B 352 io.tlbreplay(idx) := true.B 353 } 354 io.ptw.req(idx).bits.vpn := get_pn(req_out(idx).vaddr) 355 io.ptw.req(idx).bits.s2xlate := req_s2xlate 356 io.ptw.req(idx).bits.getGpa := req_need_gpa && hitVec(idx) 357 io.ptw.req(idx).bits.memidx := req_out(idx).memidx 358 } 359 360 def handle_block(idx: Int): Unit = { 361 // three valid: 1.if exist a entry; 2.if sent to ptw; 3.unset resp.valid 362 io.requestor(idx).req.ready := !req_out_v(idx) || io.requestor(idx).resp.fire 363 // req_out_v for if there is a request, may long latency, fixme 364 365 // miss request entries 366 val req_need_gpa = hasGpf(idx) 367 val miss_req_vpn = get_pn(req_out(idx).vaddr) 368 val miss_req_memidx = req_out(idx).memidx 369 val miss_req_s2xlate = Wire(UInt(2.W)) 370 miss_req_s2xlate := MuxCase(noS2xlate, Seq( 371 (!(virt_out(idx) || req_out(idx).hyperinst)) -> noS2xlate, 372 (csr.vsatp.mode =/= 0.U && csr.hgatp.mode =/= 0.U) -> allStage, 373 (csr.vsatp.mode === 0.U) -> onlyStage2, 374 (csr.hgatp.mode === 0.U || req_need_gpa) -> onlyStage1 375 )) 376 val miss_req_s2xlate_reg = RegEnable(miss_req_s2xlate, io.ptw.req(idx).fire) 377 val hasS2xlate = miss_req_s2xlate_reg =/= noS2xlate 378 val onlyS2 = miss_req_s2xlate_reg === onlyStage2 379 val hit_s1 = io.ptw.resp.bits.s1.hit(miss_req_vpn, Mux(hasS2xlate, io.csr.vsatp.asid, io.csr.satp.asid), io.csr.hgatp.vmid, allType = true, false, hasS2xlate) 380 val hit_s2 = io.ptw.resp.bits.s2.hit(miss_req_vpn, io.csr.hgatp.vmid) 381 val hit = Mux(onlyS2, hit_s2, hit_s1) && io.ptw.resp.valid && miss_req_s2xlate_reg === io.ptw.resp.bits.s2xlate 382 383 val new_coming_valid = WireInit(false.B) 384 new_coming_valid := req_in(idx).fire && !req_in(idx).bits.kill && !flush_pipe(idx) 385 val new_coming = GatedValidRegNext(new_coming_valid) 386 val miss_wire = new_coming && missVec(idx) 387 val miss_v = ValidHoldBypass(miss_wire, resp(idx).fire, flush_pipe(idx)) 388 val miss_req_v = ValidHoldBypass(miss_wire || (miss_v && flush_mmu && !mmu_flush_pipe), 389 io.ptw.req(idx).fire || resp(idx).fire, flush_pipe(idx)) 390 391 // when ptw resp, check if hit, reset miss_v, resp to lsu/ifu 392 resp(idx).valid := req_out_v(idx) && !(miss_v && portTranslateEnable(idx)) 393 when (io.ptw.resp.fire && hit && req_out_v(idx) && portTranslateEnable(idx)) { 394 val stage1 = io.ptw.resp.bits.s1 395 val stage2 = io.ptw.resp.bits.s2 396 val s2xlate = io.ptw.resp.bits.s2xlate 397 resp(idx).valid := true.B 398 resp(idx).bits.miss := false.B 399 val s1_paddr = Cat(stage1.genPPN(get_pn(req_out(idx).vaddr)), get_off(req_out(idx).vaddr)) 400 val s2_paddr = Cat(stage2.genPPNS2(get_pn(req_out(idx).vaddr)), get_off(req_out(idx).vaddr)) 401 for (d <- 0 until nRespDups) { 402 resp(idx).bits.paddr(d) := Mux(s2xlate =/= noS2xlate, s2_paddr, s1_paddr) 403 resp(idx).bits.gpaddr(d) := s1_paddr 404 pbmt_check(idx, d, io.ptw.resp.bits.s1.entry.pbmt, io.ptw.resp.bits.s2.entry.pbmt, s2xlate) 405 perm_check(stage1, req_out(idx).cmd, idx, d, stage2, req_out(idx).hlvx, s2xlate) 406 } 407 pmp_check(resp(idx).bits.paddr(0), req_out(idx).size, req_out(idx).cmd, idx) 408 409 // NOTE: the unfiltered req would be handled by Repeater 410 } 411 assert(RegNext(!resp(idx).valid || resp(idx).ready, true.B), "when tlb resp valid, ready should be true, must") 412 assert(RegNext(req_out_v(idx) || !(miss_v || miss_req_v), true.B), "when not req_out_v, should not set miss_v/miss_req_v") 413 414 val ptw_req = io.ptw.req(idx) 415 ptw_req.valid := miss_req_v 416 ptw_req.bits.vpn := miss_req_vpn 417 ptw_req.bits.s2xlate := miss_req_s2xlate 418 ptw_req.bits.getGpa := req_need_gpa && hitVec(idx) 419 ptw_req.bits.memidx := miss_req_memidx 420 421 io.tlbreplay(idx) := false.B 422 423 // NOTE: when flush pipe, tlb should abandon last req 424 // however, some outside modules like icache, dont care flushPipe, and still waiting for tlb resp 425 // just resp valid and raise page fault to go through. The pipe(ifu) will abandon it. 426 if (!q.outsideRecvFlush) { 427 when (req_out_v(idx) && flush_pipe(idx) && portTranslateEnable(idx)) { 428 resp(idx).valid := true.B 429 for (d <- 0 until nRespDups) { 430 resp(idx).bits.pbmt(d) := 0.U 431 resp(idx).bits.excp(d).pf.ld := true.B // sfence happened, pf for not to use this addr 432 resp(idx).bits.excp(d).pf.st := true.B 433 resp(idx).bits.excp(d).pf.instr := true.B 434 } 435 } 436 } 437 } 438 439 // when ptw resp, tlb at refill_idx maybe set to miss by force. 440 // Bypass ptw resp to check. 441 def ptw_resp_bypass(vpn: UInt, s2xlate: UInt) = { 442 // TODO: RegNext enable: ptw.resp.valid 443 val hasS2xlate = s2xlate =/= noS2xlate 444 val onlyS2 = s2xlate === onlyStage2 445 val onlyS1 = s2xlate === onlyStage1 446 val s2xlate_hit = s2xlate === ptw.resp.bits.s2xlate 447 val resp_hit = ptw.resp.bits.hit(vpn, io.csr.satp.asid, io.csr.vsatp.asid, io.csr.hgatp.vmid, true, false) 448 val p_hit = GatedValidRegNext(resp_hit && io.ptw.resp.fire && s2xlate_hit) 449 val ppn_s1 = ptw.resp.bits.s1.genPPN(vpn) 450 val gvpn = Mux(onlyS2, vpn, ppn_s1) 451 val ppn_s2 = ptw.resp.bits.s2.genPPNS2(gvpn) 452 val p_ppn = RegEnable(Mux(s2xlate === onlyStage2 || s2xlate === allStage, ppn_s2, ppn_s1), io.ptw.resp.fire) 453 val p_pbmt = RegEnable(ptw.resp.bits.s1.entry.pbmt,io.ptw.resp.fire) 454 val p_perm = RegEnable(ptwresp_to_tlbperm(ptw.resp.bits.s1), io.ptw.resp.fire) 455 val p_gvpn = RegEnable(Mux(onlyS2, ptw.resp.bits.s2.entry.tag, ppn_s1), io.ptw.resp.fire) 456 val p_g_pbmt = RegEnable(ptw.resp.bits.s2.entry.pbmt,io.ptw.resp.fire) 457 val p_g_perm = RegEnable(hptwresp_to_tlbperm(ptw.resp.bits.s2), io.ptw.resp.fire) 458 val p_s2xlate = RegEnable(ptw.resp.bits.s2xlate, io.ptw.resp.fire) 459 val p_s1_level = RegEnable(ptw.resp.bits.s1.entry.level.get, io.ptw.resp.fire) 460 val p_s1_isLeaf = RegEnable(ptw.resp.bits.s1.isLeaf(), io.ptw.resp.fire) 461 val p_s1_isFakePte = RegEnable(ptw.resp.bits.s1.isFakePte(), io.ptw.resp.fire) 462 (p_hit, p_ppn, p_pbmt, p_perm, p_gvpn, p_g_pbmt, p_g_perm, p_s2xlate, p_s1_level, p_s1_isLeaf, p_s1_isFakePte) 463 } 464 465 // assert 466 for(i <- 0 until Width) { 467 TimeOutAssert(req_out_v(i) && !resp(i).valid, timeOutThreshold, s"{q.name} port{i} long time no resp valid.") 468 } 469 470 // perf event 471 val result_ok = req_in.map(a => GatedValidRegNext(a.fire)) 472 val perfEvents = 473 Seq( 474 ("access", PopCount((0 until Width).map{i => if (Block(i)) io.requestor(i).req.fire else portTranslateEnable(i) && result_ok(i) })), 475 ("miss ", PopCount((0 until Width).map{i => if (Block(i)) portTranslateEnable(i) && result_ok(i) && missVec(i) else ptw.req(i).fire })), 476 ) 477 generatePerfEvent() 478 479 // perf log 480 for (i <- 0 until Width) { 481 if (Block(i)) { 482 XSPerfAccumulate(s"access${i}",result_ok(i) && portTranslateEnable(i)) 483 XSPerfAccumulate(s"miss${i}", result_ok(i) && missVec(i)) 484 } else { 485 XSPerfAccumulate("first_access" + Integer.toString(i, 10), result_ok(i) && portTranslateEnable(i) && RegEnable(req(i).bits.debug.isFirstIssue, req(i).valid)) 486 XSPerfAccumulate("access" + Integer.toString(i, 10), result_ok(i) && portTranslateEnable(i)) 487 XSPerfAccumulate("first_miss" + Integer.toString(i, 10), result_ok(i) && portTranslateEnable(i) && missVec(i) && RegEnable(req(i).bits.debug.isFirstIssue, req(i).valid)) 488 XSPerfAccumulate("miss" + Integer.toString(i, 10), result_ok(i) && portTranslateEnable(i) && missVec(i)) 489 } 490 } 491 XSPerfAccumulate("ptw_resp_count", ptw.resp.fire) 492 XSPerfAccumulate("ptw_resp_pf_count", ptw.resp.fire && ptw.resp.bits.s1.pf) 493 494 // Log 495 for(i <- 0 until Width) { 496 XSDebug(req(i).valid, p"req(${i.U}): (${req(i).valid} ${req(i).ready}) ${req(i).bits}\n") 497 XSDebug(resp(i).valid, p"resp(${i.U}): (${resp(i).valid} ${resp(i).ready}) ${resp(i).bits}\n") 498 } 499 500 XSDebug(io.sfence.valid, p"Sfence: ${io.sfence}\n") 501 XSDebug(ParallelOR(req_out_v) || ptw.resp.valid, p"vmEnable:${vmEnable} hit:${Binary(VecInit(hitVec).asUInt)} miss:${Binary(VecInit(missVec).asUInt)}\n") 502 for (i <- ptw.req.indices) { 503 XSDebug(ptw.req(i).fire, p"L2TLB req:${ptw.req(i).bits}\n") 504 } 505 XSDebug(ptw.resp.valid, p"L2TLB resp:${ptw.resp.bits} (v:${ptw.resp.valid}r:${ptw.resp.ready}) \n") 506 507 println(s"${q.name}: page: ${q.NWays} ${q.Associative} ${q.Replacer.get}") 508 509 if (env.EnableDifftest) { 510 for (i <- 0 until Width) { 511 val pf = io.requestor(i).resp.bits.excp(0).pf.instr || io.requestor(i).resp.bits.excp(0).pf.st || io.requestor(i).resp.bits.excp(0).pf.ld 512 val gpf = io.requestor(i).resp.bits.excp(0).gpf.instr || io.requestor(i).resp.bits.excp(0).gpf.st || io.requestor(i).resp.bits.excp(0).gpf.ld 513 val af = io.requestor(i).resp.bits.excp(0).af.instr || io.requestor(i).resp.bits.excp(0).af.st || io.requestor(i).resp.bits.excp(0).af.ld 514 val difftest = DifftestModule(new DiffL1TLBEvent) 515 difftest.coreid := io.hartId 516 difftest.valid := RegNext(io.requestor(i).req.fire) && !io.requestor(i).req_kill && io.requestor(i).resp.fire && !io.requestor(i).resp.bits.miss && !pf && !af && !gpf && portTranslateEnable(i) 517 if (!Seq("itlb", "ldtlb", "sttlb").contains(q.name)) { 518 difftest.valid := false.B 519 } 520 difftest.index := TLBDiffId(p(XSCoreParamsKey).HartId).U 521 difftest.vpn := RegEnable(get_pn(req_in(i).bits.vaddr), req_in(i).valid) 522 difftest.ppn := get_pn(io.requestor(i).resp.bits.paddr(0)) 523 difftest.satp := Cat(io.csr.satp.mode, io.csr.satp.asid, io.csr.satp.ppn) 524 difftest.vsatp := Cat(io.csr.vsatp.mode, io.csr.vsatp.asid, io.csr.vsatp.ppn) 525 difftest.hgatp := Cat(io.csr.hgatp.mode, io.csr.hgatp.vmid, io.csr.hgatp.ppn) 526 val req_need_gpa = gpf 527 val req_s2xlate = Wire(UInt(2.W)) 528 req_s2xlate := MuxCase(noS2xlate, Seq( 529 (!RegNext(virt_in || req_in(i).bits.hyperinst)) -> noS2xlate, 530 (vsatp.mode =/= 0.U && hgatp.mode =/= 0.U) -> allStage, 531 (vsatp.mode === 0.U) -> onlyStage2, 532 (hgatp.mode === 0.U || req_need_gpa) -> onlyStage1 533 )) 534 difftest.s2xlate := req_s2xlate 535 } 536 } 537} 538 539object TLBDiffId { 540 var i: Int = 0 541 var lastHartId: Int = -1 542 def apply(hartId: Int): Int = { 543 if (lastHartId != hartId) { 544 i = 0 545 lastHartId = hartId 546 } 547 i += 1 548 i - 1 549 } 550} 551 552class TLBNonBlock(Width: Int, nRespDups: Int = 1, q: TLBParameters)(implicit p: Parameters) extends TLB(Width, nRespDups, Seq.fill(Width)(false), q) 553class TLBBLock(Width: Int, nRespDups: Int = 1, q: TLBParameters)(implicit p: Parameters) extends TLB(Width, nRespDups, Seq.fill(Width)(true), q) 554 555class TlbReplace(Width: Int, q: TLBParameters)(implicit p: Parameters) extends TlbModule { 556 val io = IO(new TlbReplaceIO(Width, q)) 557 558 if (q.Associative == "fa") { 559 val re = ReplacementPolicy.fromString(q.Replacer, q.NWays) 560 re.access(io.page.access.map(_.touch_ways)) 561 io.page.refillIdx := re.way 562 } else { // set-acco && plru 563 val re = ReplacementPolicy.fromString(q.Replacer, q.NSets, q.NWays) 564 re.access(io.page.access.map(_.sets), io.page.access.map(_.touch_ways)) 565 io.page.refillIdx := { if (q.NWays == 1) 0.U else re.way(io.page.chosen_set) } 566 } 567} 568