1/*************************************************************************************** 2* Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences 3* Copyright (c) 2020-2021 Peng Cheng Laboratory 4* 5* XiangShan is licensed under Mulan PSL v2. 6* You can use this software according to the terms and conditions of the Mulan PSL v2. 7* You may obtain a copy of Mulan PSL v2 at: 8* http://license.coscl.org.cn/MulanPSL2 9* 10* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, 11* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, 12* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. 13* 14* See the Mulan PSL v2 for more details. 15***************************************************************************************/ 16 17package xiangshan.cache.mmu 18 19import chipsalliance.rocketchip.config.Parameters 20import chisel3._ 21import chisel3.util._ 22import xiangshan._ 23import utils._ 24import xiangshan.backend.roq.RoqPtr 25import xiangshan.backend.fu.util.HasCSRConst 26 27 28 29class TLB(Width: Int, isDtlb: Boolean)(implicit p: Parameters) extends TlbModule with HasCSRConst{ 30 val io = IO(new TlbIO(Width)) 31 32 val req = io.requestor.map(_.req) 33 val resp = io.requestor.map(_.resp) 34 val ptw = io.ptw 35 36 val sfence = io.sfence 37 val csr = io.csr 38 val satp = csr.satp 39 val priv = csr.priv 40 val ifecth = if (isDtlb) false.B else true.B 41 val mode = if (isDtlb) priv.dmode else priv.imode 42 // val vmEnable = satp.mode === 8.U // && (mode < ModeM) // FIXME: fix me when boot xv6/linux... 43 val vmEnable = if(EnbaleTlbDebug) (satp.mode === 8.U) 44 else (satp.mode === 8.U && (mode < ModeM)) 45 46 val reqAddr = req.map(_.bits.vaddr.asTypeOf(vaBundle)) 47 val cmd = req.map(_.bits.cmd) 48 val valid = req.map(_.valid) 49 50 def widthMapSeq[T <: Seq[Data]](f: Int => T) = (0 until Width).map(f) 51 def widthMap[T <: Data](f: Int => T) = (0 until Width).map(f) 52 53 // Normal page && Super page 54 val nv = RegInit(VecInit(Seq.fill(TlbEntrySize)(false.B))) 55 val nMeta = Module(new CAMTemplate(UInt(vpnLen.W), TlbEntrySize, Width + 1)).io 56 val nData = Reg(Vec(TlbEntrySize, new TlbData(false))) 57 val sv = RegInit(VecInit(Seq.fill(TlbSPEntrySize)(false.B))) 58 val sMeta = Reg(Vec(TlbSPEntrySize, new TlbSPMeta)) 59 val sData = Reg(Vec(TlbSPEntrySize, new TlbData(true))) 60 val v = nv ++ sv 61 val data = nData ++ sData 62 val g = VecInit(data.map(_.perm.g)) 63 val pf = VecInit(data.zip(v).map{ case(e, vi) => e.perm.pf & vi }) 64 65 /** 66 * PTW refill 67 */ 68 val refill = ptw.resp.fire() && !sfence.valid 69 70 val normalReplacer = if (isDtlb) Some("random") else Some("plru") 71 val superReplacer = if (isDtlb) Some("random") else Some("plru") 72 val nReplace = ReplacementPolicy.fromString(normalReplacer, TlbEntrySize) 73 val sReplace = ReplacementPolicy.fromString(superReplacer, TlbSPEntrySize) 74 val nRefillIdx = replaceWrapper(nv, nReplace.way) 75 val sRefillIdx = replaceWrapper(sv, sReplace.way) 76 77 nMeta.w := DontCare 78 nMeta.w.valid := false.B 79 when (refill) { 80 val resp = ptw.resp.bits 81 when (resp.entry.level.getOrElse(0.U) === 2.U) { 82 val refillIdx = nRefillIdx 83 refillIdx.suggestName(s"NormalRefillIdx") 84 85 nv(refillIdx) := true.B 86 nMeta.w.bits.index := nRefillIdx 87 nMeta.w.bits.data := resp.entry.tag 88 nMeta.w.valid := true.B 89 nData(refillIdx).apply( 90 ppn = resp.entry.ppn, 91 level = resp.entry.level.getOrElse(0.U), 92 perm = VecInit(resp.entry.perm.getOrElse(0.U)).asUInt, 93 pf = resp.pf 94 ) 95 nReplace.access(nRefillIdx) 96 XSDebug(p"Refill normal: idx:${refillIdx} entry:${resp.entry} pf:${resp.pf}\n") 97 }.otherwise { 98 val refillIdx = sRefillIdx 99 refillIdx.suggestName(s"SuperRefillIdx") 100 101 val dup = Cat(sv.zip(sMeta).map{ case (v, m) => 102 v && m.hit(resp.entry.tag) 103 }).orR // NOTE: may have long latency, RegNext it 104 105 when (!dup) { 106 sv(refillIdx) := true.B 107 sMeta(refillIdx).apply( 108 vpn = resp.entry.tag, 109 level = resp.entry.level.getOrElse(0.U) 110 ) 111 sData(refillIdx).apply( 112 ppn = resp.entry.ppn, 113 level = resp.entry.level.getOrElse(0.U), 114 perm = VecInit(resp.entry.perm.getOrElse(0.U)).asUInt, 115 pf = resp.pf 116 ) 117 sReplace.access(sRefillIdx) 118 XSDebug(p"Refill superpage: idx:${refillIdx} entry:${resp.entry} pf:${resp.pf}\n") 119 } 120 } 121 } 122 123 /** 124 * L1 TLB read 125 */ 126 val sfenceVpn = sfence.bits.addr.asTypeOf(vaBundle).vpn 127 for (i <- 0 until Width) { 128 nMeta.r.req(i) := io.requestor(i).req.bits.vaddr.asTypeOf(vaBundle).vpn 129 } 130 nMeta.r.req(Width) := sfenceVpn 131 132 val nRefillMask = Mux(refill, UIntToOH(nRefillIdx)(TlbEntrySize-1, 0), 0.U).asBools 133 val sRefillMask = Mux(refill, UIntToOH(sRefillIdx)(TlbSPEntrySize-1, 0), 0.U).asBools 134 def TLBNormalRead(i: Int) = { 135 val entryHitVec = ( 136 if (isDtlb) 137 VecInit(nMeta.r.resp(i).zip(nRefillMask).map{ case (e, m) => ~m && e } ++ 138 sMeta.zip(sRefillMask).map{ case (e,m) => ~m && e.hit(reqAddr(i).vpn) }) 139 else 140 VecInit(nMeta.r.resp(i) ++ sMeta.map(_.hit(reqAddr(i).vpn/*, satp.asid*/))) 141 ) 142 143 val reqAddrReg = if (isDtlb) RegNext(reqAddr(i)) else reqAddr(i) 144 val cmdReg = if (isDtlb) RegNext(cmd(i)) else cmd(i) 145 val validReg = if (isDtlb) RegNext(valid(i)) else valid(i) 146 val entryHitVecReg = if (isDtlb) RegNext(entryHitVec) else entryHitVec 147 entryHitVecReg.suggestName(s"entryHitVecReg_${i}") 148 149 val hitVec = VecInit((v zip entryHitVecReg).map{ case (a,b) => a&b }) 150 val pfHitVec = VecInit((pf zip entryHitVecReg).map{ case (a,b) => a&b }) 151 val pfArray = ParallelOR(pfHitVec).asBool && validReg && vmEnable 152 val hit = ParallelOR(hitVec).asBool && validReg && vmEnable && ~pfArray 153 val miss = !hit && validReg && vmEnable && ~pfArray 154 val hitppn = ParallelMux(hitVec zip data.map(_.genPPN(reqAddrReg.vpn))) 155 val hitPerm = ParallelMux(hitVec zip data.map(_.perm)) 156 157 hitVec.suggestName(s"hitVec_${i}") 158 pfHitVec.suggestName(s"pfHitVec_${i}") 159 hit.suggestName(s"hit_${i}") 160 miss.suggestName(s"miss_${i}") 161 hitppn.suggestName(s"hitppn_${i}") 162 hitPerm.suggestName(s"hitPerm_${i}") 163 164 if (!isDtlb) { // NOTE: only support one access 165 val hitVecUInt = hitVec.asUInt 166 XSDebug(hitVecUInt.orR, p"HitVecUInt:${Hexadecimal(hitVecUInt)}\n") 167 when (Cat(hitVecUInt(TlbEntrySize-1, 0)).orR && validReg && vmEnable) { 168 nReplace.access(OHToUInt(hitVecUInt(TlbEntrySize-1, 0))) 169 XSDebug(p"Normal Page Access: ${Hexadecimal(OHToUInt(hitVecUInt(TlbEntrySize-1, 0)))}\n") 170 } 171 when (Cat(hitVecUInt(TlbEntrySize + TlbSPEntrySize - 1, TlbEntrySize)).orR && validReg && vmEnable) { 172 sReplace.access(OHToUInt(hitVecUInt(TlbEntrySize + TlbSPEntrySize - 1, TlbEntrySize))) 173 XSDebug(p"Super Page Access: ${Hexadecimal(OHToUInt(hitVecUInt(TlbEntrySize + TlbSPEntrySize - 1, TlbEntrySize)))}\n") 174 } 175 } 176 177 XSDebug(valid(i), p"(${i.U}) entryHit:${Hexadecimal(entryHitVec.asUInt)}\n") 178 XSDebug(validReg, p"(${i.U}) entryHitReg:${Hexadecimal(entryHitVecReg.asUInt)} hitVec:${Hexadecimal(hitVec.asUInt)} pfHitVec:${Hexadecimal(pfHitVec.asUInt)} pfArray:${Hexadecimal(pfArray.asUInt)} hit:${hit} miss:${miss} hitppn:${Hexadecimal(hitppn)} hitPerm:${hitPerm}\n") 179 180 // resp // TODO: A/D has not being concerned 181 val paddr = Cat(hitppn, reqAddrReg.off) 182 val vaddr = SignExt(req(i).bits.vaddr, PAddrBits) 183 184 req(i).ready := resp(i).ready 185 resp(i).valid := validReg 186 resp(i).bits.paddr := Mux(vmEnable, paddr, if (isDtlb) RegNext(vaddr) else vaddr) 187 resp(i).bits.miss := miss 188 resp(i).bits.ptwBack := io.ptw.resp.fire() 189 190 val perm = hitPerm // NOTE: given the excp, the out module choose one to use? 191 val update = false.B && hit && (!hitPerm.a || !hitPerm.d && TlbCmd.isWrite(cmdReg)) // update A/D through exception 192 val modeCheck = !(mode === ModeU && !perm.u || mode === ModeS && perm.u && (!priv.sum || ifecth)) 193 val ldPf = (pfArray && TlbCmd.isRead(cmdReg) && true.B /*!isAMO*/) || hit && !(modeCheck && (perm.r || priv.mxr && perm.x)) && (TlbCmd.isRead(cmdReg) && true.B/*!isAMO*/) // TODO: handle isAMO 194 val stPf = (pfArray && TlbCmd.isWrite(cmdReg) || false.B /*isAMO*/ ) || hit && !(modeCheck && perm.w) && (TlbCmd.isWrite(cmdReg) || false.B/*TODO isAMO. */) 195 val instrPf = (pfArray && TlbCmd.isExec(cmdReg)) || hit && !(modeCheck && perm.x) && TlbCmd.isExec(cmdReg) 196 resp(i).bits.excp.pf.ld := ldPf || update 197 resp(i).bits.excp.pf.st := stPf || update 198 resp(i).bits.excp.pf.instr := instrPf || update 199 200 // if vmenable, use pre-calcuated pma check result 201 resp(i).bits.mmio := Mux(TlbCmd.isExec(cmdReg), !perm.pi, !perm.pd) 202 resp(i).bits.excp.af.ld := Mux(TlbCmd.isAtom(cmdReg), !perm.pa, !perm.pr) && TlbCmd.isRead(cmdReg) 203 resp(i).bits.excp.af.st := Mux(TlbCmd.isAtom(cmdReg), !perm.pa, !perm.pw) && TlbCmd.isWrite(cmdReg) 204 resp(i).bits.excp.af.instr := Mux(TlbCmd.isAtom(cmdReg), false.B, !perm.pe) 205 206 // if !vmenable, check pma 207 val (pmaMode, accessWidth) = AddressSpace.memmapAddrMatch(resp(i).bits.paddr) 208 when(!vmEnable){ 209 resp(i).bits.mmio := Mux(TlbCmd.isExec(cmdReg), !PMAMode.icache(pmaMode), !PMAMode.dcache(pmaMode)) 210 resp(i).bits.excp.af.ld := Mux(TlbCmd.isAtom(cmdReg), !PMAMode.atomic(pmaMode), !PMAMode.read(pmaMode)) && TlbCmd.isRead(cmdReg) 211 resp(i).bits.excp.af.st := Mux(TlbCmd.isAtom(cmdReg), !PMAMode.atomic(pmaMode), !PMAMode.write(pmaMode)) && TlbCmd.isWrite(cmdReg) 212 resp(i).bits.excp.af.instr := Mux(TlbCmd.isAtom(cmdReg), false.B, !PMAMode.execute(pmaMode)) 213 } 214 215 // TODO: MMIO check 216 217 (hit, miss, hitVec, validReg) 218 } 219 220 val readResult = (0 until Width).map(TLBNormalRead(_)) 221 val hitVec = readResult.map(res => res._1) 222 val missVec = readResult.map(res => res._2) 223 val hitVecVec = readResult.map(res => res._3) 224 val validRegVec = readResult.map(res => res._4) 225 226 for (i <- 0 until Width) { 227 io.ptw.req(i).valid := validRegVec(i) && missVec(i) && !RegNext(refill) 228 io.ptw.req(i).bits.vpn := RegNext(reqAddr(i).vpn) 229 } 230 io.ptw.resp.ready := true.B 231 232 // val tooManyPf = PopCount(pf) > 5.U 233 // when (tooManyPf) { // when too much pf, just clear 234 // XSDebug(p"Too many pf just flush all the pf v:${Hexadecimal(VecInit(v).asUInt)} pf:${Hexadecimal(pf.asUInt)}\n") 235 // v.zipWithIndex.map{ case (a, i) => a := a & !pf(i) } 236 // } 237 238 // sfence (flush) 239 val sfenceHit = nMeta.r.resp(Width) ++ sMeta.map(_.hit(sfenceVpn)) 240 when (sfence.valid) { 241 when (sfence.bits.rs1) { // virtual address *.rs1 <- (rs1===0.U) 242 when (sfence.bits.rs2) { // asid, but i do not want to support asid, *.rs2 <- (rs2===0.U) 243 // all addr and all asid 244 v.map(_ := false.B) 245 }.otherwise { 246 // all addr but specific asid 247 v.zipWithIndex.map{ case (a,i) => a := a & g(i) } 248 } 249 }.otherwise { 250 when (sfence.bits.rs2) { 251 // specific addr but all asid 252 v.zipWithIndex.map{ case (a,i) => a := a & !sfenceHit(i) } 253 }.otherwise { 254 // specific addr and specific asid 255 v.zipWithIndex.map{ case (a,i) => a := a & !sfenceHit(i) && !g(i) } 256 } 257 } 258 } 259 260 if (isDtlb) { 261 for (i <- 0 until Width) { 262 XSPerfAccumulate("first_access" + Integer.toString(i, 10), validRegVec(i) && vmEnable && RegNext(req(i).bits.debug.isFirstIssue)) 263 XSPerfAccumulate("access" + Integer.toString(i, 10), validRegVec(i) && vmEnable) 264 } 265 for (i <- 0 until Width) { 266 XSPerfAccumulate("first_miss" + Integer.toString(i, 10), validRegVec(i) && vmEnable && missVec(i) && RegNext(req(i).bits.debug.isFirstIssue)) 267 XSPerfAccumulate("miss" + Integer.toString(i, 10), validRegVec(i) && vmEnable && missVec(i)) 268 } 269 } else { 270 // NOTE: ITLB is blocked, so every resp will be valid only when hit 271 // every req will be ready only when hit 272 XSPerfAccumulate("access", io.requestor(0).req.fire() && vmEnable) 273 XSPerfAccumulate("miss", ptw.req(0).fire()) 274 } 275 //val reqCycleCnt = Reg(UInt(16.W)) 276 //reqCycleCnt := reqCycleCnt + BoolStopWatch(ptw.req(0).fire(), ptw.resp.fire || sfence.valid) 277 //XSPerfAccumulate("ptw_req_count", ptw.req.fire()) 278 //XSPerfAccumulate("ptw_req_cycle", Mux(ptw.resp.fire(), reqCycleCnt, 0.U)) 279 XSPerfAccumulate("ptw_resp_count", ptw.resp.fire()) 280 XSPerfAccumulate("ptw_resp_pf_count", ptw.resp.fire() && ptw.resp.bits.pf) 281 for (i <- 0 until TlbEntrySize) { 282 val indexHitVec = hitVecVec.zip(validRegVec).map{ case (h, v) => h(i) && v } 283 XSPerfAccumulate(s"NormalAccessIndex${i}", Mux(vmEnable, PopCount(indexHitVec), 0.U)) 284 } 285 for (i <- 0 until TlbSPEntrySize) { 286 val indexHitVec = hitVecVec.zip(validRegVec).map{ case (h, v) => h(i + TlbEntrySize) && v } 287 XSPerfAccumulate(s"SuperAccessIndex${i}", Mux(vmEnable, PopCount(indexHitVec), 0.U)) 288 } 289 for (i <- 0 until TlbEntrySize) { 290 XSPerfAccumulate(s"NormalRefillIndex${i}", refill && ptw.resp.bits.entry.level.getOrElse(0.U) === 2.U && i.U === nRefillIdx) 291 } 292 for (i <- 0 until TlbSPEntrySize) { 293 XSPerfAccumulate(s"SuperRefillIndex${i}", refill && ptw.resp.bits.entry.level.getOrElse(0.U) =/= 2.U && i.U === sRefillIdx) 294 } 295 296 // Log 297 for(i <- 0 until Width) { 298 XSDebug(req(i).valid, p"req(${i.U}): (${req(i).valid} ${req(i).ready}) ${req(i).bits}\n") 299 XSDebug(resp(i).valid, p"resp(${i.U}): (${resp(i).valid} ${resp(i).ready}) ${resp(i).bits}\n") 300 } 301 302 XSDebug(sfence.valid, p"Sfence: ${sfence}\n") 303 XSDebug(ParallelOR(valid)|| ptw.resp.valid, p"CSR: ${csr}\n") 304 XSDebug(ParallelOR(valid) || ptw.resp.valid, p"vmEnable:${vmEnable} hit:${Binary(VecInit(hitVec).asUInt)} miss:${Binary(VecInit(missVec).asUInt)} v:${Hexadecimal(VecInit(v).asUInt)} pf:${Hexadecimal(pf.asUInt)}\n") 305 for (i <- ptw.req.indices) { 306 XSDebug(ptw.req(i).fire(), p"PTW req:${ptw.req(i).bits}\n") 307 } 308 XSDebug(ptw.resp.valid, p"PTW resp:${ptw.resp.bits} (v:${ptw.resp.valid}r:${ptw.resp.ready}) \n") 309 310// // NOTE: just for simple tlb debug, comment it after tlb's debug 311 // assert(!io.ptw.resp.valid || io.ptw.resp.bits.entry.tag === io.ptw.resp.bits.entry.ppn, "Simple tlb debug requires vpn === ppn") 312} 313 314object TLB { 315 def apply 316 ( 317 in: Seq[BlockTlbRequestIO], 318 sfence: SfenceBundle, 319 csr: TlbCsrBundle, 320 width: Int, 321 isDtlb: Boolean, 322 shouldBlock: Boolean 323 )(implicit p: Parameters) = { 324 require(in.length == width) 325 326 val tlb = Module(new TLB(width, isDtlb)) 327 328 tlb.io.sfence <> sfence 329 tlb.io.csr <> csr 330 331 if (!shouldBlock) { // dtlb 332 for (i <- 0 until width) { 333 tlb.io.requestor(i) <> in(i) 334 // tlb.io.requestor(i).req.valid := in(i).req.valid 335 // tlb.io.requestor(i).req.bits := in(i).req.bits 336 // in(i).req.ready := tlb.io.requestor(i).req.ready 337 338 // in(i).resp.valid := tlb.io.requestor(i).resp.valid 339 // in(i).resp.bits := tlb.io.requestor(i).resp.bits 340 // tlb.io.requestor(i).resp.ready := in(i).resp.ready 341 } 342 } else { // itlb 343 require(width == 1) 344 tlb.io.requestor(0).req.valid := in(0).req.valid 345 tlb.io.requestor(0).req.bits := in(0).req.bits 346 in(0).req.ready := !tlb.io.requestor(0).resp.bits.miss && in(0).resp.ready && tlb.io.requestor(0).req.ready 347 348 in(0).resp.valid := tlb.io.requestor(0).resp.valid && !tlb.io.requestor(0).resp.bits.miss 349 in(0).resp.bits := tlb.io.requestor(0).resp.bits 350 tlb.io.requestor(0).resp.ready := in(0).resp.ready 351 } 352 353 tlb.io.ptw 354 } 355} 356