1/*************************************************************************************** 2* Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences 3* Copyright (c) 2020-2021 Peng Cheng Laboratory 4* 5* XiangShan is licensed under Mulan PSL v2. 6* You can use this software according to the terms and conditions of the Mulan PSL v2. 7* You may obtain a copy of Mulan PSL v2 at: 8* http://license.coscl.org.cn/MulanPSL2 9* 10* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, 11* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, 12* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. 13* 14* See the Mulan PSL v2 for more details. 15***************************************************************************************/ 16 17package xiangshan.cache.mmu 18 19import org.chipsalliance.cde.config.Parameters 20import chisel3._ 21import chisel3.experimental.ExtModule 22import chisel3.util._ 23import xiangshan._ 24import xiangshan.cache.{HasDCacheParameters, MemoryOpConstants} 25import utils._ 26import utility._ 27import freechips.rocketchip.diplomacy.{IdRange, LazyModule, LazyModuleImp} 28import freechips.rocketchip.tilelink._ 29import xiangshan.backend.fu.{PMP, PMPChecker, PMPReqBundle, PMPRespBundle} 30import xiangshan.backend.fu.util.HasCSRConst 31import difftest._ 32 33class L2TLB()(implicit p: Parameters) extends LazyModule with HasPtwConst { 34 override def shouldBeInlined: Boolean = false 35 36 val node = TLClientNode(Seq(TLMasterPortParameters.v1( 37 clients = Seq(TLMasterParameters.v1( 38 "ptw", 39 sourceId = IdRange(0, MemReqWidth) 40 )), 41 requestFields = Seq(ReqSourceField()) 42 ))) 43 44 lazy val module = new L2TLBImp(this) 45} 46 47class L2TLBImp(outer: L2TLB)(implicit p: Parameters) extends PtwModule(outer) with HasCSRConst with HasPerfEvents { 48 49 val (mem, edge) = outer.node.out.head 50 51 val io = IO(new L2TLBIO) 52 val difftestIO = IO(new Bundle() { 53 val ptwResp = Output(Bool()) 54 val ptwAddr = Output(UInt(64.W)) 55 val ptwData = Output(Vec(4, UInt(64.W))) 56 }) 57 58 /* Ptw processes multiple requests 59 * Divide Ptw procedure into two stages: cache access ; mem access if cache miss 60 * miss queue itlb dtlb 61 * | | | 62 * ------arbiter------ 63 * | 64 * l1 - l2 - l3 - sp 65 * | 66 * ------------------------------------------- 67 * miss | queue | hit 68 * [][][][][][] | 69 * | | 70 * state machine accessing mem | 71 * | | 72 * ---------------arbiter--------------------- 73 * | | 74 * itlb dtlb 75 */ 76 77 difftestIO <> DontCare 78 79 val sfence_tmp = DelayN(io.sfence, 1) 80 val csr_tmp = DelayN(io.csr.tlb, 1) 81 val sfence_dup = Seq.fill(9)(RegNext(sfence_tmp)) 82 val csr_dup = Seq.fill(8)(RegNext(csr_tmp)) // TODO: add csr_modified? 83 val satp = csr_dup(0).satp 84 val vsatp = csr_dup(0).vsatp 85 val hgatp = csr_dup(0).hgatp 86 val priv = csr_dup(0).priv 87 val mPBMTE = csr_dup(0).mPBMTE 88 val hPBMTE = csr_dup(0).hPBMTE 89 val flush = sfence_dup(0).valid || satp.changed || vsatp.changed || hgatp.changed 90 91 val pmp = Module(new PMP()) 92 val pmp_check = VecInit(Seq.fill(3)(Module(new PMPChecker(lgMaxSize = 3, sameCycle = true)).io)) 93 pmp.io.distribute_csr := io.csr.distribute_csr 94 pmp_check.foreach(_.check_env.apply(ModeS, pmp.io.pmp, pmp.io.pma)) 95 96 val missQueue = Module(new L2TlbMissQueue) 97 val cache = Module(new PtwCache) 98 val ptw = Module(new PTW) 99 val hptw = Module(new HPTW) 100 val llptw = Module(new LLPTW) 101 val blockmq = Module(new BlockHelper(3)) 102 val arb1 = Module(new Arbiter(new PtwReq, PtwWidth)) 103 val arb2 = Module(new Arbiter(new L2TlbWithHptwIdBundle, ((if (l2tlbParams.enablePrefetch) 4 else 3) + (if(HasHExtension) 1 else 0)))) 104 val hptw_req_arb = Module(new Arbiter(new Bundle { 105 val id = UInt(log2Up(l2tlbParams.llptwsize).W) 106 val source = UInt(bSourceWidth.W) 107 val gvpn = UInt(gvpnLen.W) 108 }, 2)) 109 val hptw_resp_arb = Module(new Arbiter(new Bundle { 110 val resp = new HptwResp() 111 val id = UInt(log2Up(l2tlbParams.llptwsize).W) 112 }, 2)) 113 val outArb = (0 until PtwWidth).map(i => Module(new Arbiter(new Bundle { 114 val s2xlate = UInt(2.W) 115 val s1 = new PtwSectorResp () 116 val s2 = new HptwResp() 117 }, 1)).io) 118 val mergeArb = (0 until PtwWidth).map(i => Module(new Arbiter(new Bundle { 119 val s2xlate = UInt(2.W) 120 val s1 = new PtwMergeResp() 121 val s2 = new HptwResp() 122 }, 3)).io) 123 val outArbCachePort = 0 124 val outArbFsmPort = 1 125 val outArbMqPort = 2 126 127 // hptw arb input port 128 val InHptwArbPTWPort = 0 129 val InHptwArbLLPTWPort = 1 130 hptw_req_arb.io.in(InHptwArbPTWPort).valid := ptw.io.hptw.req.valid 131 hptw_req_arb.io.in(InHptwArbPTWPort).bits.gvpn := ptw.io.hptw.req.bits.gvpn 132 hptw_req_arb.io.in(InHptwArbPTWPort).bits.id := ptw.io.hptw.req.bits.id 133 hptw_req_arb.io.in(InHptwArbPTWPort).bits.source := ptw.io.hptw.req.bits.source 134 ptw.io.hptw.req.ready := hptw_req_arb.io.in(InHptwArbPTWPort).ready 135 136 hptw_req_arb.io.in(InHptwArbLLPTWPort).valid := llptw.io.hptw.req.valid 137 hptw_req_arb.io.in(InHptwArbLLPTWPort).bits.gvpn := llptw.io.hptw.req.bits.gvpn 138 hptw_req_arb.io.in(InHptwArbLLPTWPort).bits.id := llptw.io.hptw.req.bits.id 139 hptw_req_arb.io.in(InHptwArbLLPTWPort).bits.source := llptw.io.hptw.req.bits.source 140 llptw.io.hptw.req.ready := hptw_req_arb.io.in(InHptwArbLLPTWPort).ready 141 142 // arb2 input port 143 val InArbHPTWPort = 0 144 val InArbPTWPort = 1 145 val InArbMissQueuePort = 2 146 val InArbTlbPort = 3 147 val InArbPrefetchPort = 4 148 // NOTE: when cache out but miss and ptw doesnt accept, 149 arb1.io.in <> VecInit(io.tlb.map(_.req(0))) 150 151 val tlbCounter = RegInit(0.U(log2Ceil(MissQueueSize + 1).W)) 152 val reqVec = WireInit(VecInit(Seq.fill(PtwWidth)(false.B))) 153 val respVec = WireInit(VecInit(Seq.fill(PtwWidth)(false.B))) 154 155 for (i <- 0 until PtwWidth) { 156 when (io.tlb(i).req(0).fire) { 157 reqVec(i) := true.B 158 } 159 when (io.tlb(i).resp.fire) { 160 respVec(i) := true.B 161 } 162 } 163 164 when (flush) { 165 tlbCounter := 0.U 166 } .otherwise { 167 tlbCounter := tlbCounter + PopCount(reqVec) - PopCount(respVec) 168 } 169 XSError(!(tlbCounter >= 0.U && tlbCounter <= MissQueueSize.U), s"l2tlb full!") 170 171 arb2.io.in(InArbPTWPort).valid := ptw.io.llptw.valid 172 arb2.io.in(InArbPTWPort).bits.req_info := ptw.io.llptw.bits.req_info 173 arb2.io.in(InArbPTWPort).bits.isHptwReq := false.B 174 arb2.io.in(InArbPTWPort).bits.isLLptw := false.B 175 arb2.io.in(InArbPTWPort).bits.hptwId := DontCare 176 ptw.io.llptw.ready := arb2.io.in(InArbPTWPort).ready 177 block_decoupled(missQueue.io.out, arb2.io.in(InArbMissQueuePort), Mux(missQueue.io.out.bits.isLLptw, !llptw.io.in.ready, !ptw.io.req.ready)) 178 179 arb2.io.in(InArbTlbPort).valid := arb1.io.out.fire 180 arb2.io.in(InArbTlbPort).bits.req_info.vpn := arb1.io.out.bits.vpn 181 arb2.io.in(InArbTlbPort).bits.req_info.s2xlate := arb1.io.out.bits.s2xlate 182 arb2.io.in(InArbTlbPort).bits.req_info.source := arb1.io.chosen 183 arb2.io.in(InArbTlbPort).bits.isHptwReq := false.B 184 arb2.io.in(InArbTlbPort).bits.isLLptw := false.B 185 arb2.io.in(InArbTlbPort).bits.hptwId := DontCare 186 // 1. arb1 and arb2 are both comb logic, so ready can work just the same cycle 187 // 2. arb1 can send one req at most in a cycle, so do not need to write 188 // "tlbCounter <= (MissQueueSize - 2).U" 189 arb1.io.out.ready := arb2.io.in(InArbTlbPort).ready && tlbCounter < MissQueueSize.U 190 191 arb2.io.in(InArbHPTWPort).valid := hptw_req_arb.io.out.valid 192 arb2.io.in(InArbHPTWPort).bits.req_info.vpn := hptw_req_arb.io.out.bits.gvpn 193 arb2.io.in(InArbHPTWPort).bits.req_info.s2xlate := onlyStage2 194 arb2.io.in(InArbHPTWPort).bits.req_info.source := hptw_req_arb.io.out.bits.source 195 arb2.io.in(InArbHPTWPort).bits.isHptwReq := true.B 196 arb2.io.in(InArbHPTWPort).bits.isLLptw := false.B 197 arb2.io.in(InArbHPTWPort).bits.hptwId := hptw_req_arb.io.out.bits.id 198 hptw_req_arb.io.out.ready := arb2.io.in(InArbHPTWPort).ready 199 val hartId = p(XSCoreParamsKey).HartId 200 if (l2tlbParams.enablePrefetch) { 201 val prefetch = Module(new L2TlbPrefetch()) 202 val recv = cache.io.resp 203 // NOTE: 1. prefetch doesn't gen prefetch 2. req from mq doesn't gen prefetch 204 // NOTE: 1. miss req gen prefetch 2. hit but prefetched gen prefetch 205 prefetch.io.in.valid := recv.fire && !from_pre(recv.bits.req_info.source) && (!recv.bits.hit || 206 recv.bits.prefetch) && recv.bits.isFirst 207 prefetch.io.in.bits.vpn := recv.bits.req_info.vpn 208 prefetch.io.sfence := sfence_dup(0) 209 prefetch.io.csr := csr_dup(0) 210 arb2.io.in(InArbPrefetchPort) <> prefetch.io.out 211 212 val isWriteL2TlbPrefetchTable = Constantin.createRecord(s"isWriteL2TlbPrefetchTable$hartId") 213 val L2TlbPrefetchTable = ChiselDB.createTable(s"L2TlbPrefetch_hart$hartId", new L2TlbPrefetchDB) 214 val L2TlbPrefetchDB = Wire(new L2TlbPrefetchDB) 215 L2TlbPrefetchDB.vpn := prefetch.io.out.bits.req_info.vpn 216 L2TlbPrefetchTable.log(L2TlbPrefetchDB, isWriteL2TlbPrefetchTable.orR && prefetch.io.out.fire, "L2TlbPrefetch", clock, reset) 217 } 218 arb2.io.out.ready := cache.io.req.ready 219 220 221 val mq_arb = Module(new Arbiter(new L2TlbWithHptwIdBundle, 2)) 222 mq_arb.io.in(0).valid := cache.io.resp.valid && !cache.io.resp.bits.hit && 223 !from_pre(cache.io.resp.bits.req_info.source) && !cache.io.resp.bits.isHptwReq && // hptw reqs are not sent to missqueue 224 (cache.io.resp.bits.bypassed || ( 225 ((!cache.io.resp.bits.toFsm.l1Hit || cache.io.resp.bits.toFsm.stage1Hit) && !cache.io.resp.bits.isHptwReq && (cache.io.resp.bits.isFirst || !ptw.io.req.ready)) // send to ptw, is first or ptw is busy; 226 || (cache.io.resp.bits.toFsm.l1Hit && !llptw.io.in.ready) // send to llptw, llptw is full 227 )) 228 229 mq_arb.io.in(0).bits.req_info := cache.io.resp.bits.req_info 230 mq_arb.io.in(0).bits.isHptwReq := false.B 231 mq_arb.io.in(0).bits.hptwId := DontCare 232 mq_arb.io.in(0).bits.isLLptw := cache.io.resp.bits.toFsm.l1Hit 233 mq_arb.io.in(1).bits.req_info := llptw.io.cache.bits 234 mq_arb.io.in(1).bits.isHptwReq := false.B 235 mq_arb.io.in(1).bits.hptwId := DontCare 236 mq_arb.io.in(1).bits.isLLptw := false.B 237 mq_arb.io.in(1).valid := llptw.io.cache.valid 238 llptw.io.cache.ready := mq_arb.io.in(1).ready 239 missQueue.io.in <> mq_arb.io.out 240 missQueue.io.sfence := sfence_dup(6) 241 missQueue.io.csr := csr_dup(5) 242 243 blockmq.io.start := missQueue.io.out.fire 244 blockmq.io.enable := ptw.io.req.fire 245 246 llptw.io.in.valid := cache.io.resp.valid && 247 !cache.io.resp.bits.hit && 248 cache.io.resp.bits.toFsm.l1Hit && 249 !cache.io.resp.bits.bypassed && 250 !cache.io.resp.bits.isHptwReq 251 llptw.io.in.bits.req_info := cache.io.resp.bits.req_info 252 llptw.io.in.bits.ppn := cache.io.resp.bits.toFsm.ppn 253 llptw.io.sfence := sfence_dup(1) 254 llptw.io.csr := csr_dup(1) 255 val llptw_stage1 = Reg(Vec(l2tlbParams.llptwsize, new PtwMergeResp())) 256 when(llptw.io.in.fire){ 257 llptw_stage1(llptw.io.mem.enq_ptr) := cache.io.resp.bits.stage1 258 } 259 260 cache.io.req.valid := arb2.io.out.fire 261 cache.io.req.bits.req_info := arb2.io.out.bits.req_info 262 cache.io.req.bits.isFirst := (arb2.io.chosen =/= InArbMissQueuePort.U && !arb2.io.out.bits.isHptwReq) 263 cache.io.req.bits.isHptwReq := arb2.io.out.bits.isHptwReq 264 cache.io.req.bits.hptwId := arb2.io.out.bits.hptwId 265 cache.io.req.bits.bypassed.map(_ := false.B) 266 cache.io.sfence := sfence_dup(2) 267 cache.io.csr := csr_dup(2) 268 cache.io.sfence_dup.zip(sfence_dup.drop(2).take(4)).map(s => s._1 := s._2) 269 cache.io.csr_dup.zip(csr_dup.drop(2).take(3)).map(c => c._1 := c._2) 270 cache.io.resp.ready := MuxCase(mq_arb.io.in(0).ready || ptw.io.req.ready, Seq( 271 (!cache.io.resp.bits.hit && cache.io.resp.bits.isHptwReq) -> hptw.io.req.ready, 272 (cache.io.resp.bits.hit && cache.io.resp.bits.isHptwReq) -> hptw_resp_arb.io.in(HptwRespArbCachePort).ready, 273 cache.io.resp.bits.hit -> outReady(cache.io.resp.bits.req_info.source, outArbCachePort), 274 (cache.io.resp.bits.toFsm.l1Hit && !cache.io.resp.bits.bypassed && llptw.io.in.ready) -> llptw.io.in.ready, 275 (cache.io.resp.bits.bypassed || cache.io.resp.bits.isFirst) -> mq_arb.io.in(0).ready 276 )) 277 278 // NOTE: missQueue req has higher priority 279 ptw.io.req.valid := cache.io.resp.valid && !cache.io.resp.bits.hit && !cache.io.resp.bits.toFsm.l1Hit && 280 !cache.io.resp.bits.bypassed && 281 !cache.io.resp.bits.isFirst && 282 !cache.io.resp.bits.isHptwReq 283 ptw.io.req.bits.req_info := cache.io.resp.bits.req_info 284 if (EnableSv48) { 285 ptw.io.req.bits.l3Hit.get := cache.io.resp.bits.toFsm.l3Hit.get 286 } 287 ptw.io.req.bits.l2Hit := cache.io.resp.bits.toFsm.l2Hit 288 ptw.io.req.bits.ppn := cache.io.resp.bits.toFsm.ppn 289 ptw.io.req.bits.stage1Hit := cache.io.resp.bits.toFsm.stage1Hit 290 ptw.io.req.bits.stage1 := cache.io.resp.bits.stage1 291 ptw.io.sfence := sfence_dup(7) 292 ptw.io.csr := csr_dup(6) 293 ptw.io.resp.ready := outReady(ptw.io.resp.bits.source, outArbFsmPort) 294 295 hptw.io.req.valid := cache.io.resp.valid && !cache.io.resp.bits.hit && cache.io.resp.bits.isHptwReq 296 hptw.io.req.bits.gvpn := cache.io.resp.bits.req_info.vpn 297 hptw.io.req.bits.id := cache.io.resp.bits.toHptw.id 298 hptw.io.req.bits.source := cache.io.resp.bits.req_info.source 299 if (EnableSv48) { 300 hptw.io.req.bits.l3Hit.get := cache.io.resp.bits.toHptw.l3Hit.get 301 } 302 hptw.io.req.bits.l2Hit := cache.io.resp.bits.toHptw.l2Hit 303 hptw.io.req.bits.l1Hit := cache.io.resp.bits.toHptw.l1Hit 304 hptw.io.req.bits.ppn := cache.io.resp.bits.toHptw.ppn 305 hptw.io.req.bits.bypassed := cache.io.resp.bits.toHptw.bypassed 306 hptw.io.sfence := sfence_dup(8) 307 hptw.io.csr := csr_dup(7) 308 // mem req 309 def blockBytes_align(addr: UInt) = { 310 Cat(addr(PAddrBits - 1, log2Up(l2tlbParams.blockBytes)), 0.U(log2Up(l2tlbParams.blockBytes).W)) 311 } 312 def addr_low_from_vpn(vpn: UInt) = { 313 vpn(log2Ceil(l2tlbParams.blockBytes)-log2Ceil(XLEN/8)-1, 0) 314 } 315 def addr_low_from_paddr(paddr: UInt) = { 316 paddr(log2Up(l2tlbParams.blockBytes)-1, log2Up(XLEN/8)) 317 } 318 def from_llptw(id: UInt) = { 319 id < l2tlbParams.llptwsize.U 320 } 321 def from_ptw(id: UInt) = { 322 id === l2tlbParams.llptwsize.U 323 } 324 def from_hptw(id: UInt) = { 325 id === l2tlbParams.llptwsize.U + 1.U 326 } 327 val waiting_resp = RegInit(VecInit(Seq.fill(MemReqWidth)(false.B))) 328 val flush_latch = RegInit(VecInit(Seq.fill(MemReqWidth)(false.B))) 329 val hptw_bypassed = RegInit(false.B) 330 for (i <- waiting_resp.indices) { 331 assert(!flush_latch(i) || waiting_resp(i)) // when sfence_latch wait for mem resp, waiting_resp should be true 332 } 333 334 val llptw_out = llptw.io.out 335 val llptw_mem = llptw.io.mem 336 llptw_mem.flush_latch := flush_latch.take(l2tlbParams.llptwsize) 337 llptw_mem.req_mask := waiting_resp.take(l2tlbParams.llptwsize) 338 ptw.io.mem.mask := waiting_resp.apply(l2tlbParams.llptwsize) 339 hptw.io.mem.mask := waiting_resp.apply(l2tlbParams.llptwsize + 1) 340 341 val mem_arb = Module(new Arbiter(new L2TlbMemReqBundle(), 3)) 342 mem_arb.io.in(0) <> ptw.io.mem.req 343 mem_arb.io.in(1) <> llptw_mem.req 344 mem_arb.io.in(2) <> hptw.io.mem.req 345 mem_arb.io.out.ready := mem.a.ready && !flush 346 347 // // assert, should not send mem access at same addr for twice. 348 // val last_resp_vpn = RegEnable(cache.io.refill.bits.req_info_dup(0).vpn, cache.io.refill.valid) 349 // val last_resp_s2xlate = RegEnable(cache.io.refill.bits.req_info_dup(0).s2xlate, cache.io.refill.valid) 350 // val last_resp_level = RegEnable(cache.io.refill.bits.level_dup(0), cache.io.refill.valid) 351 // val last_resp_v = RegInit(false.B) 352 // val last_has_invalid = !Cat(cache.io.refill.bits.ptes.asTypeOf(Vec(blockBits/XLEN, UInt(XLEN.W))).map(a => a(0))).andR || cache.io.refill.bits.sel_pte_dup(0).asTypeOf(new PteBundle).isAf() 353 // when (cache.io.refill.valid) { last_resp_v := !last_has_invalid} 354 // when (flush) { last_resp_v := false.B } 355 // XSError(last_resp_v && cache.io.refill.valid && 356 // (cache.io.refill.bits.req_info_dup(0).vpn === last_resp_vpn) && 357 // (cache.io.refill.bits.level_dup(0) === last_resp_level) && 358 // (cache.io.refill.bits.req_info_dup(0).s2xlate === last_resp_s2xlate), 359 // "l2tlb should not access mem at same addr for twice") 360 // // ATTENTION: this may wrongly assert when: a ptes is l2, last part is valid, 361 // // but the current part is invalid, so one more mem access happened 362 // // If this happened, remove the assert. 363 364 val req_addr_low = Reg(Vec(MemReqWidth, UInt((log2Up(l2tlbParams.blockBytes)-log2Up(XLEN/8)).W))) 365 366 when (llptw.io.in.fire) { 367 // when enq miss queue, set the req_addr_low to receive the mem resp data part 368 req_addr_low(llptw_mem.enq_ptr) := addr_low_from_vpn(llptw.io.in.bits.req_info.vpn) 369 } 370 when (mem_arb.io.out.fire) { 371 req_addr_low(mem_arb.io.out.bits.id) := addr_low_from_paddr(mem_arb.io.out.bits.addr) 372 waiting_resp(mem_arb.io.out.bits.id) := true.B 373 hptw_bypassed := from_hptw(mem_arb.io.out.bits.id) && mem_arb.io.out.bits.hptw_bypassed 374 } 375 // mem read 376 val memRead = edge.Get( 377 fromSource = mem_arb.io.out.bits.id, 378 // toAddress = memAddr(log2Up(CacheLineSize / 2 / 8) - 1, 0), 379 toAddress = blockBytes_align(mem_arb.io.out.bits.addr), 380 lgSize = log2Up(l2tlbParams.blockBytes).U 381 )._2 382 mem.a.bits := memRead 383 mem.a.valid := mem_arb.io.out.valid && !flush 384 mem.a.bits.user.lift(ReqSourceKey).foreach(_ := MemReqSource.PTW.id.U) 385 mem.d.ready := true.B 386 // mem -> data buffer 387 val refill_data = RegInit(VecInit.fill(blockBits / l1BusDataWidth)(0.U(l1BusDataWidth.W))) 388 val refill_helper = edge.firstlastHelper(mem.d.bits, mem.d.fire) 389 val mem_resp_done = refill_helper._3 390 val mem_resp_from_llptw = from_llptw(mem.d.bits.source) 391 val mem_resp_from_ptw = from_ptw(mem.d.bits.source) 392 val mem_resp_from_hptw = from_hptw(mem.d.bits.source) 393 when (mem.d.valid) { 394 assert(mem.d.bits.source < MemReqWidth.U) 395 refill_data(refill_helper._4) := mem.d.bits.data 396 } 397 // refill_data_tmp is the wire fork of refill_data, but one cycle earlier 398 val refill_data_tmp = WireInit(refill_data) 399 refill_data_tmp(refill_helper._4) := mem.d.bits.data 400 401 // save only one pte for each id 402 // (miss queue may can't resp to tlb with low latency, it should have highest priority, but diffcult to design cache) 403 val resp_pte = VecInit((0 until MemReqWidth).map(i => 404 if (i == l2tlbParams.llptwsize + 1) {RegEnable(get_part(refill_data_tmp, req_addr_low(i)), 0.U.asTypeOf(get_part(refill_data_tmp, req_addr_low(i))), mem_resp_done && mem_resp_from_hptw) } 405 else if (i == l2tlbParams.llptwsize) {RegEnable(get_part(refill_data_tmp, req_addr_low(i)), 0.U.asTypeOf(get_part(refill_data_tmp, req_addr_low(i))), mem_resp_done && mem_resp_from_ptw) } 406 else { Mux(llptw_mem.buffer_it(i), get_part(refill_data, req_addr_low(i)), RegEnable(get_part(refill_data, req_addr_low(i)), 0.U.asTypeOf(get_part(refill_data, req_addr_low(i))), llptw_mem.buffer_it(i))) } 407 // llptw could not use refill_data_tmp, because enq bypass's result works at next cycle 408 )) 409 410 // save eight ptes for each id when sector tlb 411 // (miss queue may can't resp to tlb with low latency, it should have highest priority, but diffcult to design cache) 412 val resp_pte_sector = VecInit((0 until MemReqWidth).map(i => 413 if (i == l2tlbParams.llptwsize + 1) {RegEnable(refill_data_tmp, 0.U.asTypeOf(refill_data_tmp), mem_resp_done && mem_resp_from_hptw) } 414 else if (i == l2tlbParams.llptwsize) {RegEnable(refill_data_tmp, 0.U.asTypeOf(refill_data_tmp), mem_resp_done && mem_resp_from_ptw) } 415 else { Mux(llptw_mem.buffer_it(i), refill_data, RegEnable(refill_data, 0.U.asTypeOf(refill_data), llptw_mem.buffer_it(i))) } 416 // llptw could not use refill_data_tmp, because enq bypass's result works at next cycle 417 )) 418 419 // mem -> llptw 420 llptw_mem.resp.valid := mem_resp_done && mem_resp_from_llptw 421 llptw_mem.resp.bits.id := DataHoldBypass(mem.d.bits.source, mem.d.valid) 422 llptw_mem.resp.bits.value := DataHoldBypass(refill_data_tmp.asUInt, mem.d.valid) 423 // mem -> ptw 424 ptw.io.mem.resp.valid := mem_resp_done && mem_resp_from_ptw 425 ptw.io.mem.resp.bits := resp_pte.apply(l2tlbParams.llptwsize) 426 // mem -> hptw 427 hptw.io.mem.resp.valid := mem_resp_done && mem_resp_from_hptw 428 hptw.io.mem.resp.bits := resp_pte.apply(l2tlbParams.llptwsize + 1) 429 // mem -> cache 430 val refill_from_llptw = mem_resp_from_llptw 431 val refill_from_ptw = mem_resp_from_ptw 432 val refill_from_hptw = mem_resp_from_hptw 433 val refill_level = Mux(refill_from_llptw, 0.U, Mux(refill_from_ptw, RegEnable(ptw.io.refill.level, 0.U, ptw.io.mem.req.fire), RegEnable(hptw.io.refill.level, 0.U, hptw.io.mem.req.fire))) 434 val refill_valid = mem_resp_done && !flush && !flush_latch(mem.d.bits.source) && !hptw_bypassed 435 436 cache.io.refill.valid := GatedValidRegNext(refill_valid, false.B) 437 cache.io.refill.bits.ptes := refill_data.asUInt 438 cache.io.refill.bits.req_info_dup.map(_ := RegEnable(Mux(refill_from_llptw, llptw_mem.refill, Mux(refill_from_ptw, ptw.io.refill.req_info, hptw.io.refill.req_info)), refill_valid)) 439 cache.io.refill.bits.level_dup.map(_ := RegEnable(refill_level, refill_valid)) 440 cache.io.refill.bits.levelOH(refill_level, refill_valid) 441 cache.io.refill.bits.sel_pte_dup.map(_ := RegEnable(sel_data(refill_data_tmp.asUInt, req_addr_low(mem.d.bits.source)), refill_valid)) 442 443 if (env.EnableDifftest) { 444 val difftest_ptw_addr = RegInit(VecInit(Seq.fill(MemReqWidth)(0.U(PAddrBits.W)))) 445 when (mem.a.valid) { 446 difftest_ptw_addr(mem.a.bits.source) := mem.a.bits.address 447 } 448 449 val difftest = DifftestModule(new DiffRefillEvent, dontCare = true) 450 difftest.coreid := io.hartId 451 difftest.index := 2.U 452 difftest.valid := cache.io.refill.valid 453 difftest.addr := difftest_ptw_addr(RegEnable(mem.d.bits.source, mem.d.valid)) 454 difftest.data := refill_data.asTypeOf(difftest.data) 455 difftest.idtfr := DontCare 456 } 457 458 if (env.EnableDifftest) { 459 for (i <- 0 until PtwWidth) { 460 val difftest = DifftestModule(new DiffL2TLBEvent) 461 difftest.coreid := io.hartId 462 difftest.valid := io.tlb(i).resp.fire && !io.tlb(i).resp.bits.s1.af && !io.tlb(i).resp.bits.s2.gaf 463 difftest.index := i.U 464 difftest.vpn := Cat(io.tlb(i).resp.bits.s1.entry.tag, 0.U(sectortlbwidth.W)) 465 difftest.pbmt := io.tlb(i).resp.bits.s1.entry.pbmt 466 difftest.g_pbmt := io.tlb(i).resp.bits.s2.entry.pbmt 467 for (j <- 0 until tlbcontiguous) { 468 difftest.ppn(j) := Cat(io.tlb(i).resp.bits.s1.entry.ppn, io.tlb(i).resp.bits.s1.ppn_low(j)) 469 difftest.valididx(j) := io.tlb(i).resp.bits.s1.valididx(j) 470 difftest.pteidx(j) := io.tlb(i).resp.bits.s1.pteidx(j) 471 } 472 difftest.perm := io.tlb(i).resp.bits.s1.entry.perm.getOrElse(0.U.asTypeOf(new PtePermBundle)).asUInt 473 difftest.level := io.tlb(i).resp.bits.s1.entry.level.getOrElse(0.U.asUInt) 474 difftest.pf := io.tlb(i).resp.bits.s1.pf 475 difftest.satp := Cat(io.csr.tlb.satp.mode, io.csr.tlb.satp.asid, io.csr.tlb.satp.ppn) 476 difftest.vsatp := Cat(io.csr.tlb.vsatp.mode, io.csr.tlb.vsatp.asid, io.csr.tlb.vsatp.ppn) 477 difftest.hgatp := Cat(io.csr.tlb.hgatp.mode, io.csr.tlb.hgatp.vmid, io.csr.tlb.hgatp.ppn) 478 difftest.gvpn := io.tlb(i).resp.bits.s2.entry.tag 479 difftest.g_perm := io.tlb(i).resp.bits.s2.entry.perm.getOrElse(0.U.asTypeOf(new PtePermBundle)).asUInt 480 difftest.g_level := io.tlb(i).resp.bits.s2.entry.level.getOrElse(0.U.asUInt) 481 difftest.s2ppn := io.tlb(i).resp.bits.s2.entry.ppn 482 difftest.gpf := io.tlb(i).resp.bits.s2.gpf 483 difftest.s2xlate := io.tlb(i).resp.bits.s2xlate 484 } 485 } 486 487 // pmp 488 pmp_check(0).req <> ptw.io.pmp.req 489 ptw.io.pmp.resp <> pmp_check(0).resp 490 pmp_check(1).req <> llptw.io.pmp.req 491 llptw.io.pmp.resp <> pmp_check(1).resp 492 pmp_check(2).req <> hptw.io.pmp.req 493 hptw.io.pmp.resp <> pmp_check(2).resp 494 495 llptw_out.ready := outReady(llptw_out.bits.req_info.source, outArbMqPort) 496 497 // hptw and page cache -> ptw and llptw 498 val HptwRespArbCachePort = 0 499 val HptwRespArbHptw = 1 500 hptw_resp_arb.io.in(HptwRespArbCachePort).valid := cache.io.resp.valid && cache.io.resp.bits.hit && cache.io.resp.bits.isHptwReq 501 hptw_resp_arb.io.in(HptwRespArbCachePort).bits.id := cache.io.resp.bits.toHptw.id 502 hptw_resp_arb.io.in(HptwRespArbCachePort).bits.resp := cache.io.resp.bits.toHptw.resp 503 hptw_resp_arb.io.in(HptwRespArbHptw).valid := hptw.io.resp.valid 504 hptw_resp_arb.io.in(HptwRespArbHptw).bits.id := hptw.io.resp.bits.id 505 hptw_resp_arb.io.in(HptwRespArbHptw).bits.resp := hptw.io.resp.bits.resp 506 hptw.io.resp.ready := hptw_resp_arb.io.in(HptwRespArbHptw).ready 507 508 ptw.io.hptw.resp.valid := hptw_resp_arb.io.out.valid && hptw_resp_arb.io.out.bits.id === FsmReqID.U 509 ptw.io.hptw.resp.bits.h_resp := hptw_resp_arb.io.out.bits.resp 510 llptw.io.hptw.resp.valid := hptw_resp_arb.io.out.valid && hptw_resp_arb.io.out.bits.id =/= FsmReqID.U 511 llptw.io.hptw.resp.bits.id := hptw_resp_arb.io.out.bits.id 512 llptw.io.hptw.resp.bits.h_resp := hptw_resp_arb.io.out.bits.resp 513 hptw_resp_arb.io.out.ready := true.B 514 515 // Timing: Maybe need to do some optimization or even add one more cycle 516 for (i <- 0 until PtwWidth) { 517 mergeArb(i).in(outArbCachePort).valid := cache.io.resp.valid && cache.io.resp.bits.hit && cache.io.resp.bits.req_info.source===i.U && !cache.io.resp.bits.isHptwReq 518 mergeArb(i).in(outArbCachePort).bits.s2xlate := cache.io.resp.bits.req_info.s2xlate 519 mergeArb(i).in(outArbCachePort).bits.s1 := cache.io.resp.bits.stage1 520 mergeArb(i).in(outArbCachePort).bits.s2 := cache.io.resp.bits.toHptw.resp 521 mergeArb(i).in(outArbFsmPort).valid := ptw.io.resp.valid && ptw.io.resp.bits.source===i.U 522 mergeArb(i).in(outArbFsmPort).bits.s2xlate := ptw.io.resp.bits.s2xlate 523 mergeArb(i).in(outArbFsmPort).bits.s1 := ptw.io.resp.bits.resp 524 mergeArb(i).in(outArbFsmPort).bits.s2 := ptw.io.resp.bits.h_resp 525 mergeArb(i).in(outArbMqPort).valid := llptw_out.valid && llptw_out.bits.req_info.source===i.U 526 mergeArb(i).in(outArbMqPort).bits.s2xlate := llptw_out.bits.req_info.s2xlate 527 mergeArb(i).in(outArbMqPort).bits.s1 := Mux( 528 llptw_out.bits.first_s2xlate_fault, llptw_stage1(llptw_out.bits.id), 529 contiguous_pte_to_merge_ptwResp( 530 resp_pte_sector(llptw_out.bits.id).asUInt, llptw_out.bits.req_info.vpn, llptw_out.bits.af, 531 true, s2xlate = llptw_out.bits.req_info.s2xlate, mPBMTE = mPBMTE, hPBMTE = hPBMTE, gpf = llptw_out.bits.h_resp.gpf 532 ) 533 ) 534 mergeArb(i).in(outArbMqPort).bits.s2 := llptw_out.bits.h_resp 535 mergeArb(i).out.ready := outArb(i).in(0).ready 536 } 537 538 for (i <- 0 until PtwWidth) { 539 outArb(i).in(0).valid := mergeArb(i).out.valid 540 outArb(i).in(0).bits.s2xlate := mergeArb(i).out.bits.s2xlate 541 outArb(i).in(0).bits.s1 := merge_ptwResp_to_sector_ptwResp(mergeArb(i).out.bits.s1) 542 outArb(i).in(0).bits.s2 := mergeArb(i).out.bits.s2 543 } 544 545 // io.tlb.map(_.resp) <> outArb.map(_.out) 546 io.tlb.map(_.resp).zip(outArb.map(_.out)).map{ 547 case (resp, out) => resp <> out 548 } 549 550 // sfence 551 when (flush) { 552 for (i <- 0 until MemReqWidth) { 553 when (waiting_resp(i)) { 554 flush_latch(i) := true.B 555 } 556 } 557 } 558 // mem -> control signal 559 // waiting_resp and sfence_latch will be reset when mem_resp_done 560 when (mem_resp_done) { 561 waiting_resp(mem.d.bits.source) := false.B 562 flush_latch(mem.d.bits.source) := false.B 563 } 564 565 def block_decoupled[T <: Data](source: DecoupledIO[T], sink: DecoupledIO[T], block_signal: Bool) = { 566 sink.valid := source.valid && !block_signal 567 source.ready := sink.ready && !block_signal 568 sink.bits := source.bits 569 } 570 571 def get_part(data: Vec[UInt], index: UInt): UInt = { 572 val inner_data = data.asTypeOf(Vec(data.getWidth / XLEN, UInt(XLEN.W))) 573 inner_data(index) 574 } 575 576 // not_super means that this is a normal page 577 // valididx(i) will be all true when super page to be convenient for l1 tlb matching 578 def contiguous_pte_to_merge_ptwResp(pte: UInt, vpn: UInt, af: Bool, af_first: Boolean, s2xlate: UInt, mPBMTE: Bool, hPBMTE: Bool, not_super: Boolean = true, gpf: Bool) : PtwMergeResp = { 579 assert(tlbcontiguous == 8, "Only support tlbcontiguous = 8!") 580 val ptw_merge_resp = Wire(new PtwMergeResp()) 581 val hasS2xlate = s2xlate =/= noS2xlate 582 val pbmte = Mux(s2xlate === onlyStage1 || s2xlate === allStage, hPBMTE, mPBMTE) 583 for (i <- 0 until tlbcontiguous) { 584 val pte_in = pte(64 * i + 63, 64 * i).asTypeOf(new PteBundle()) 585 val ptw_resp = Wire(new PtwMergeEntry(tagLen = sectorvpnLen, hasPerm = true, hasLevel = true, hasNapot = true)) 586 ptw_resp.ppn := pte_in.getPPN()(ptePPNLen - 1, sectortlbwidth) 587 ptw_resp.ppn_low := pte_in.getPPN()(sectortlbwidth - 1, 0) 588 ptw_resp.level.map(_ := 0.U) 589 ptw_resp.pbmt := pte_in.pbmt 590 ptw_resp.n.map(_ := pte_in.n) 591 ptw_resp.perm.map(_ := pte_in.getPerm()) 592 ptw_resp.tag := vpn(vpnLen - 1, sectortlbwidth) 593 ptw_resp.pf := (if (af_first) !af else true.B) && (pte_in.isPf(0.U, pbmte) || !pte_in.isLeaf()) 594 ptw_resp.af := (if (!af_first) pte_in.isPf(0.U, pbmte) else true.B) && (af || (Mux(s2xlate === allStage, false.B, pte_in.isAf()) && !(hasS2xlate && gpf))) 595 ptw_resp.v := !ptw_resp.pf 596 ptw_resp.prefetch := DontCare 597 ptw_resp.asid := Mux(hasS2xlate, vsatp.asid, satp.asid) 598 ptw_resp.vmid.map(_ := hgatp.vmid) 599 ptw_merge_resp.entry(i) := ptw_resp 600 } 601 ptw_merge_resp.pteidx := UIntToOH(vpn(sectortlbwidth - 1, 0)).asBools 602 ptw_merge_resp.not_super := not_super.B 603 ptw_merge_resp.not_merge := hasS2xlate 604 ptw_merge_resp 605 } 606 607 def merge_ptwResp_to_sector_ptwResp(pte: PtwMergeResp) : PtwSectorResp = { 608 assert(tlbcontiguous == 8, "Only support tlbcontiguous = 8!") 609 val ptw_sector_resp = Wire(new PtwSectorResp) 610 ptw_sector_resp.entry.tag := pte.entry(OHToUInt(pte.pteidx)).tag 611 ptw_sector_resp.entry.asid := pte.entry(OHToUInt(pte.pteidx)).asid 612 ptw_sector_resp.entry.vmid.map(_ := pte.entry(OHToUInt(pte.pteidx)).vmid.getOrElse(0.U)) 613 ptw_sector_resp.entry.ppn := pte.entry(OHToUInt(pte.pteidx)).ppn 614 ptw_sector_resp.entry.pbmt := pte.entry(OHToUInt(pte.pteidx)).pbmt 615 ptw_sector_resp.entry.n.map(_ := pte.entry(OHToUInt(pte.pteidx)).n.getOrElse(0.U)) 616 ptw_sector_resp.entry.perm.map(_ := pte.entry(OHToUInt(pte.pteidx)).perm.getOrElse(0.U.asTypeOf(new PtePermBundle))) 617 ptw_sector_resp.entry.level.map(_ := pte.entry(OHToUInt(pte.pteidx)).level.getOrElse(0.U(log2Up(Level + 1).W))) 618 ptw_sector_resp.entry.prefetch := pte.entry(OHToUInt(pte.pteidx)).prefetch 619 ptw_sector_resp.entry.v := pte.entry(OHToUInt(pte.pteidx)).v 620 ptw_sector_resp.af := pte.entry(OHToUInt(pte.pteidx)).af 621 ptw_sector_resp.pf := pte.entry(OHToUInt(pte.pteidx)).pf 622 ptw_sector_resp.addr_low := OHToUInt(pte.pteidx) 623 ptw_sector_resp.pteidx := pte.pteidx 624 for (i <- 0 until tlbcontiguous) { 625 val ppn_equal = pte.entry(i).ppn === pte.entry(OHToUInt(pte.pteidx)).ppn 626 val pbmt_equal = pte.entry(i).pbmt === pte.entry(OHToUInt(pte.pteidx)).pbmt 627 val n_equal = pte.entry(i).n.getOrElse(0.U) === pte.entry(OHToUInt(pte.pteidx)).n.getOrElse(0.U) 628 val perm_equal = pte.entry(i).perm.getOrElse(0.U.asTypeOf(new PtePermBundle)).asUInt === pte.entry(OHToUInt(pte.pteidx)).perm.getOrElse(0.U.asTypeOf(new PtePermBundle)).asUInt 629 val v_equal = pte.entry(i).v === pte.entry(OHToUInt(pte.pteidx)).v 630 val af_equal = pte.entry(i).af === pte.entry(OHToUInt(pte.pteidx)).af 631 val pf_equal = pte.entry(i).pf === pte.entry(OHToUInt(pte.pteidx)).pf 632 ptw_sector_resp.valididx(i) := ((ppn_equal && pbmt_equal && n_equal && perm_equal && v_equal && af_equal && pf_equal) || !pte.not_super) && !pte.not_merge 633 ptw_sector_resp.ppn_low(i) := pte.entry(i).ppn_low 634 } 635 ptw_sector_resp.valididx(OHToUInt(pte.pteidx)) := true.B 636 ptw_sector_resp 637 } 638 639 def outReady(source: UInt, port: Int): Bool = { 640 MuxLookup(source, true.B)((0 until PtwWidth).map(i => i.U -> mergeArb(i).in(port).ready)) 641 } 642 643 // debug info 644 for (i <- 0 until PtwWidth) { 645 XSDebug(p"[io.tlb(${i.U})] ${io.tlb(i)}\n") 646 } 647 XSDebug(p"[sfence] ${io.sfence}\n") 648 XSDebug(p"[io.csr.tlb] ${io.csr.tlb}\n") 649 650 for (i <- 0 until PtwWidth) { 651 XSPerfAccumulate(s"req_count${i}", io.tlb(i).req(0).fire) 652 XSPerfAccumulate(s"req_blocked_count_${i}", io.tlb(i).req(0).valid && !io.tlb(i).req(0).ready) 653 } 654 XSPerfAccumulate(s"req_blocked_by_mq", arb1.io.out.valid && missQueue.io.out.valid) 655 for (i <- 0 until (MemReqWidth + 1)) { 656 XSPerfAccumulate(s"mem_req_util${i}", PopCount(waiting_resp) === i.U) 657 } 658 XSPerfAccumulate("mem_cycle", PopCount(waiting_resp) =/= 0.U) 659 XSPerfAccumulate("mem_count", mem.a.fire) 660 for (i <- 0 until PtwWidth) { 661 XSPerfAccumulate(s"llptw_ppn_af${i}", mergeArb(i).in(outArbMqPort).valid && mergeArb(i).in(outArbMqPort).bits.s1.entry(OHToUInt(mergeArb(i).in(outArbMqPort).bits.s1.pteidx)).af && !llptw_out.bits.af) 662 XSPerfAccumulate(s"access_fault${i}", io.tlb(i).resp.fire && io.tlb(i).resp.bits.s1.af) 663 } 664 665 // print configs 666 println(s"${l2tlbParams.name}: a ptw, a llptw with size ${l2tlbParams.llptwsize}, miss queue size ${MissQueueSize} l2:${l2tlbParams.l2Size} fa l1: nSets ${l2tlbParams.l1nSets} nWays ${l2tlbParams.l1nWays} l0: ${l2tlbParams.l0nSets} nWays ${l2tlbParams.l0nWays} blockBytes:${l2tlbParams.blockBytes}") 667 668 val perfEvents = Seq(llptw, cache, ptw).flatMap(_.getPerfEvents) 669 generatePerfEvent() 670 671 val isWriteL1TlbTable = Constantin.createRecord(s"isWriteL1TlbTable$hartId") 672 val L1TlbTable = ChiselDB.createTable(s"L1Tlb_hart$hartId", new L1TlbDB) 673 val ITlbReqDB, DTlbReqDB, ITlbRespDB, DTlbRespDB = Wire(new L1TlbDB) 674 ITlbReqDB.vpn := io.tlb(0).req(0).bits.vpn 675 DTlbReqDB.vpn := io.tlb(1).req(0).bits.vpn 676 ITlbRespDB.vpn := Cat(io.tlb(0).resp.bits.s1.entry.tag, OHToUInt(io.tlb(0).resp.bits.s1.pteidx)) 677 DTlbRespDB.vpn := Cat(io.tlb(1).resp.bits.s1.entry.tag, OHToUInt(io.tlb(1).resp.bits.s1.pteidx)) 678 L1TlbTable.log(ITlbReqDB, isWriteL1TlbTable.orR && io.tlb(0).req(0).fire, "ITlbReq", clock, reset) 679 L1TlbTable.log(DTlbReqDB, isWriteL1TlbTable.orR && io.tlb(1).req(0).fire, "DTlbReq", clock, reset) 680 L1TlbTable.log(ITlbRespDB, isWriteL1TlbTable.orR && io.tlb(0).resp.fire, "ITlbResp", clock, reset) 681 L1TlbTable.log(DTlbRespDB, isWriteL1TlbTable.orR && io.tlb(1).resp.fire, "DTlbResp", clock, reset) 682 683 val isWritePageCacheTable = Constantin.createRecord(s"isWritePageCacheTable$hartId") 684 val PageCacheTable = ChiselDB.createTable(s"PageCache_hart$hartId", new PageCacheDB) 685 val PageCacheDB = Wire(new PageCacheDB) 686 PageCacheDB.vpn := Cat(cache.io.resp.bits.stage1.entry(0).tag, OHToUInt(cache.io.resp.bits.stage1.pteidx)) 687 PageCacheDB.source := cache.io.resp.bits.req_info.source 688 PageCacheDB.bypassed := cache.io.resp.bits.bypassed 689 PageCacheDB.is_first := cache.io.resp.bits.isFirst 690 PageCacheDB.prefetched := cache.io.resp.bits.stage1.entry(0).prefetch 691 PageCacheDB.prefetch := cache.io.resp.bits.prefetch 692 PageCacheDB.l2Hit := cache.io.resp.bits.toFsm.l2Hit 693 PageCacheDB.l1Hit := cache.io.resp.bits.toFsm.l1Hit 694 PageCacheDB.hit := cache.io.resp.bits.hit 695 PageCacheTable.log(PageCacheDB, isWritePageCacheTable.orR && cache.io.resp.fire, "PageCache", clock, reset) 696 697 val isWritePTWTable = Constantin.createRecord(s"isWritePTWTable$hartId") 698 val PTWTable = ChiselDB.createTable(s"PTW_hart$hartId", new PTWDB) 699 val PTWReqDB, PTWRespDB, LLPTWReqDB, LLPTWRespDB = Wire(new PTWDB) 700 PTWReqDB.vpn := ptw.io.req.bits.req_info.vpn 701 PTWReqDB.source := ptw.io.req.bits.req_info.source 702 PTWRespDB.vpn := ptw.io.refill.req_info.vpn 703 PTWRespDB.source := ptw.io.refill.req_info.source 704 LLPTWReqDB.vpn := llptw.io.in.bits.req_info.vpn 705 LLPTWReqDB.source := llptw.io.in.bits.req_info.source 706 LLPTWRespDB.vpn := llptw.io.mem.refill.vpn 707 LLPTWRespDB.source := llptw.io.mem.refill.source 708 PTWTable.log(PTWReqDB, isWritePTWTable.orR && ptw.io.req.fire, "PTWReq", clock, reset) 709 PTWTable.log(PTWRespDB, isWritePTWTable.orR && ptw.io.mem.resp.fire, "PTWResp", clock, reset) 710 PTWTable.log(LLPTWReqDB, isWritePTWTable.orR && llptw.io.in.fire, "LLPTWReq", clock, reset) 711 PTWTable.log(LLPTWRespDB, isWritePTWTable.orR && llptw.io.mem.resp.fire, "LLPTWResp", clock, reset) 712 713 val isWriteL2TlbMissQueueTable = Constantin.createRecord(s"isWriteL2TlbMissQueueTable$hartId") 714 val L2TlbMissQueueTable = ChiselDB.createTable(s"L2TlbMissQueue_hart$hartId", new L2TlbMissQueueDB) 715 val L2TlbMissQueueInDB, L2TlbMissQueueOutDB = Wire(new L2TlbMissQueueDB) 716 L2TlbMissQueueInDB.vpn := missQueue.io.in.bits.req_info.vpn 717 L2TlbMissQueueOutDB.vpn := missQueue.io.out.bits.req_info.vpn 718 L2TlbMissQueueTable.log(L2TlbMissQueueInDB, isWriteL2TlbMissQueueTable.orR && missQueue.io.in.fire, "L2TlbMissQueueIn", clock, reset) 719 L2TlbMissQueueTable.log(L2TlbMissQueueOutDB, isWriteL2TlbMissQueueTable.orR && missQueue.io.out.fire, "L2TlbMissQueueOut", clock, reset) 720} 721 722/** BlockHelper, block missqueue, not to send too many req to cache 723 * Parameter: 724 * enable: enable BlockHelper, mq should not send too many reqs 725 * start: when miss queue out fire and need, block miss queue's out 726 * block: block miss queue's out 727 * latency: last missqueue out's cache access latency 728 */ 729class BlockHelper(latency: Int)(implicit p: Parameters) extends XSModule { 730 val io = IO(new Bundle { 731 val enable = Input(Bool()) 732 val start = Input(Bool()) 733 val block = Output(Bool()) 734 }) 735 736 val count = RegInit(0.U(log2Ceil(latency).W)) 737 val valid = RegInit(false.B) 738 val work = RegInit(true.B) 739 740 io.block := valid 741 742 when (io.start && work) { valid := true.B } 743 when (valid) { count := count + 1.U } 744 when (count === (latency.U) || io.enable) { 745 valid := false.B 746 work := io.enable 747 count := 0.U 748 } 749} 750 751class PTEHelper() extends ExtModule { 752 val clock = IO(Input(Clock())) 753 val enable = IO(Input(Bool())) 754 val satp = IO(Input(UInt(64.W))) 755 val vpn = IO(Input(UInt(64.W))) 756 val pte = IO(Output(UInt(64.W))) 757 val level = IO(Output(UInt(8.W))) 758 val pf = IO(Output(UInt(8.W))) 759} 760 761class PTWDelayN[T <: Data](gen: T, n: Int, flush: Bool) extends Module { 762 val io = IO(new Bundle() { 763 val in = Input(gen) 764 val out = Output(gen) 765 val ptwflush = Input(flush.cloneType) 766 }) 767 val out = RegInit(VecInit(Seq.fill(n)(0.U.asTypeOf(gen)))) 768 val t = RegInit(VecInit(Seq.fill(n)(0.U.asTypeOf(gen)))) 769 out(0) := io.in 770 if (n == 1) { 771 io.out := out(0) 772 } else { 773 when (io.ptwflush) { 774 for (i <- 0 until n) { 775 t(i) := 0.U.asTypeOf(gen) 776 out(i) := 0.U.asTypeOf(gen) 777 } 778 io.out := 0.U.asTypeOf(gen) 779 } .otherwise { 780 for (i <- 1 until n) { 781 t(i-1) := out(i-1) 782 out(i) := t(i-1) 783 } 784 io.out := out(n-1) 785 } 786 } 787} 788 789object PTWDelayN { 790 def apply[T <: Data](in: T, n: Int, flush: Bool): T = { 791 val delay = Module(new PTWDelayN(in.cloneType, n, flush)) 792 delay.io.in := in 793 delay.io.ptwflush := flush 794 delay.io.out 795 } 796} 797 798class FakePTW()(implicit p: Parameters) extends XSModule with HasPtwConst { 799 val io = IO(new L2TLBIO) 800 val flush = VecInit(Seq.fill(PtwWidth)(false.B)) 801 flush(0) := DelayN(io.sfence.valid || io.csr.tlb.satp.changed, itlbParams.fenceDelay) 802 flush(1) := DelayN(io.sfence.valid || io.csr.tlb.satp.changed, ldtlbParams.fenceDelay) 803 for (i <- 0 until PtwWidth) { 804 val helper = Module(new PTEHelper()) 805 helper.clock := clock 806 helper.satp := io.csr.tlb.satp.ppn 807 808 if (coreParams.softPTWDelay == 1) { 809 helper.enable := io.tlb(i).req(0).fire 810 helper.vpn := io.tlb(i).req(0).bits.vpn 811 } else { 812 helper.enable := PTWDelayN(io.tlb(i).req(0).fire, coreParams.softPTWDelay - 1, flush(i)) 813 helper.vpn := PTWDelayN(io.tlb(i).req(0).bits.vpn, coreParams.softPTWDelay - 1, flush(i)) 814 } 815 816 val pte = helper.pte.asTypeOf(new PteBundle) 817 val level = helper.level 818 val pf = helper.pf 819 val empty = RegInit(true.B) 820 when (io.tlb(i).req(0).fire) { 821 empty := false.B 822 } .elsewhen (io.tlb(i).resp.fire || flush(i)) { 823 empty := true.B 824 } 825 826 io.tlb(i).req(0).ready := empty || io.tlb(i).resp.fire 827 io.tlb(i).resp.valid := PTWDelayN(io.tlb(i).req(0).fire, coreParams.softPTWDelay, flush(i)) 828 assert(!io.tlb(i).resp.valid || io.tlb(i).resp.ready) 829 io.tlb(i).resp.bits.s1.entry.tag := PTWDelayN(io.tlb(i).req(0).bits.vpn, coreParams.softPTWDelay, flush(i)) 830 io.tlb(i).resp.bits.s1.entry.pbmt := pte.pbmt 831 io.tlb(i).resp.bits.s1.entry.ppn := pte.ppn 832 io.tlb(i).resp.bits.s1.entry.perm.map(_ := pte.getPerm()) 833 io.tlb(i).resp.bits.s1.entry.level.map(_ := level) 834 io.tlb(i).resp.bits.s1.pf := pf 835 io.tlb(i).resp.bits.s1.af := DontCare // TODO: implement it 836 io.tlb(i).resp.bits.s1.entry.v := !pf 837 io.tlb(i).resp.bits.s1.entry.prefetch := DontCare 838 io.tlb(i).resp.bits.s1.entry.asid := io.csr.tlb.satp.asid 839 } 840} 841 842class L2TLBWrapper()(implicit p: Parameters) extends LazyModule with HasXSParameter { 843 override def shouldBeInlined: Boolean = false 844 val useSoftPTW = coreParams.softPTW 845 val node = if (!useSoftPTW) TLIdentityNode() else null 846 val ptw = if (!useSoftPTW) LazyModule(new L2TLB()) else null 847 if (!useSoftPTW) { 848 node := ptw.node 849 } 850 851 class L2TLBWrapperImp(wrapper: LazyModule) extends LazyModuleImp(wrapper) with HasPerfEvents { 852 val io = IO(new L2TLBIO) 853 val perfEvents = if (useSoftPTW) { 854 val fake_ptw = Module(new FakePTW()) 855 io <> fake_ptw.io 856 Seq() 857 } 858 else { 859 io <> ptw.module.io 860 ptw.module.getPerfEvents 861 } 862 generatePerfEvent() 863 } 864 865 lazy val module = new L2TLBWrapperImp(this) 866} 867