1/*************************************************************************************** 2* Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences 3* Copyright (c) 2020-2021 Peng Cheng Laboratory 4* 5* XiangShan is licensed under Mulan PSL v2. 6* You can use this software according to the terms and conditions of the Mulan PSL v2. 7* You may obtain a copy of Mulan PSL v2 at: 8* http://license.coscl.org.cn/MulanPSL2 9* 10* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, 11* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, 12* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. 13* 14* See the Mulan PSL v2 for more details. 15***************************************************************************************/ 16 17package xiangshan.cache.mmu 18 19import chipsalliance.rocketchip.config.Parameters 20import chisel3._ 21import chisel3.experimental.ExtModule 22import chisel3.util._ 23import chisel3.internal.naming.chiselName 24import xiangshan._ 25import xiangshan.cache.{HasDCacheParameters, MemoryOpConstants} 26import utils._ 27import utility._ 28import freechips.rocketchip.diplomacy.{IdRange, LazyModule, LazyModuleImp} 29import freechips.rocketchip.tilelink._ 30import xiangshan.backend.fu.{PMP, PMPChecker, PMPReqBundle, PMPRespBundle} 31import xiangshan.backend.fu.util.HasCSRConst 32import difftest._ 33 34class L2TLB()(implicit p: Parameters) extends LazyModule with HasPtwConst { 35 36 val node = TLClientNode(Seq(TLMasterPortParameters.v1( 37 clients = Seq(TLMasterParameters.v1( 38 "ptw", 39 sourceId = IdRange(0, MemReqWidth) 40 )), 41 requestFields = Seq(ReqSourceField()) 42 ))) 43 44 lazy val module = new L2TLBImp(this) 45} 46 47@chiselName 48class L2TLBImp(outer: L2TLB)(implicit p: Parameters) extends PtwModule(outer) with HasCSRConst with HasPerfEvents { 49 50 val (mem, edge) = outer.node.out.head 51 52 val io = IO(new L2TLBIO) 53 val difftestIO = IO(new Bundle() { 54 val ptwResp = Output(Bool()) 55 val ptwAddr = Output(UInt(64.W)) 56 val ptwData = Output(Vec(4, UInt(64.W))) 57 }) 58 59 /* Ptw processes multiple requests 60 * Divide Ptw procedure into two stages: cache access ; mem access if cache miss 61 * miss queue itlb dtlb 62 * | | | 63 * ------arbiter------ 64 * | 65 * l1 - l2 - l3 - sp 66 * | 67 * ------------------------------------------- 68 * miss | queue | hit 69 * [][][][][][] | 70 * | | 71 * state machine accessing mem | 72 * | | 73 * ---------------arbiter--------------------- 74 * | | 75 * itlb dtlb 76 */ 77 78 difftestIO <> DontCare 79 80 val sfence_tmp = DelayN(io.sfence, 1) 81 val csr_tmp = DelayN(io.csr.tlb, 1) 82 val sfence_dup = Seq.fill(8)(RegNext(sfence_tmp)) 83 val csr_dup = Seq.fill(7)(RegNext(csr_tmp)) 84 val satp = csr_dup(0).satp 85 val priv = csr_dup(0).priv 86 val flush = sfence_dup(0).valid || satp.changed 87 88 val pmp = Module(new PMP()) 89 val pmp_check = VecInit(Seq.fill(2)(Module(new PMPChecker(lgMaxSize = 3, sameCycle = true)).io)) 90 pmp.io.distribute_csr := io.csr.distribute_csr 91 pmp_check.foreach(_.check_env.apply(ModeS, pmp.io.pmp, pmp.io.pma)) 92 93 val missQueue = Module(new L2TlbMissQueue) 94 val cache = Module(new PtwCache) 95 val ptw = Module(new PTW) 96 val llptw = Module(new LLPTW) 97 val blockmq = Module(new BlockHelper(3)) 98 val arb1 = Module(new Arbiter(new PtwReq, PtwWidth)) 99 val arb2 = Module(new Arbiter(new Bundle { 100 val vpn = UInt(vpnLen.W) 101 val source = UInt(bSourceWidth.W) 102 }, if (l2tlbParams.enablePrefetch) 4 else 3)) 103 val outArb = (0 until PtwWidth).map(i => Module(new Arbiter(new PtwSectorResp, 1)).io) 104 val mergeArb = (0 until PtwWidth).map(i => Module(new Arbiter(new PtwMergeResp, 3)).io) 105 val outArbCachePort = 0 106 val outArbFsmPort = 1 107 val outArbMqPort = 2 108 109 // arb2 input port 110 val InArbPTWPort = 0 111 val InArbMissQueuePort = 1 112 val InArbTlbPort = 2 113 val InArbPrefetchPort = 3 114 // NOTE: when cache out but miss and ptw doesnt accept, 115 arb1.io.in <> VecInit(io.tlb.map(_.req(0))) 116 arb1.io.out.ready := arb2.io.in(InArbTlbPort).ready 117 118 arb2.io.in(InArbPTWPort).valid := ptw.io.llptw.valid 119 arb2.io.in(InArbPTWPort).bits.vpn := ptw.io.llptw.bits.req_info.vpn 120 arb2.io.in(InArbPTWPort).bits.source := ptw.io.llptw.bits.req_info.source 121 ptw.io.llptw.ready := arb2.io.in(InArbPTWPort).ready 122 block_decoupled(missQueue.io.out, arb2.io.in(InArbMissQueuePort), !ptw.io.req.ready) 123 124 arb2.io.in(InArbTlbPort).valid := arb1.io.out.valid 125 arb2.io.in(InArbTlbPort).bits.vpn := arb1.io.out.bits.vpn 126 arb2.io.in(InArbTlbPort).bits.source := arb1.io.chosen 127 if (l2tlbParams.enablePrefetch) { 128 val prefetch = Module(new L2TlbPrefetch()) 129 val recv = cache.io.resp 130 // NOTE: 1. prefetch doesn't gen prefetch 2. req from mq doesn't gen prefetch 131 // NOTE: 1. miss req gen prefetch 2. hit but prefetched gen prefetch 132 prefetch.io.in.valid := recv.fire() && !from_pre(recv.bits.req_info.source) && (!recv.bits.hit || 133 recv.bits.prefetch) && recv.bits.isFirst 134 prefetch.io.in.bits.vpn := recv.bits.req_info.vpn 135 prefetch.io.sfence := sfence_dup(0) 136 prefetch.io.csr := csr_dup(0) 137 arb2.io.in(InArbPrefetchPort) <> prefetch.io.out 138 139 val isWriteL2TlbPrefetchTable = WireInit(Constantin.createRecord("isWriteL2TlbPrefetchTable" + p(XSCoreParamsKey).HartId.toString)) 140 val L2TlbPrefetchTable = ChiselDB.createTable("L2TlbPrefetch_hart" + p(XSCoreParamsKey).HartId.toString, new L2TlbPrefetchDB) 141 val L2TlbPrefetchDB = Wire(new L2TlbPrefetchDB) 142 L2TlbPrefetchDB.vpn := prefetch.io.out.bits.vpn 143 L2TlbPrefetchTable.log(L2TlbPrefetchDB, isWriteL2TlbPrefetchTable.orR && prefetch.io.out.fire, "L2TlbPrefetch", clock, reset) 144 } 145 arb2.io.out.ready := cache.io.req.ready 146 147 148 val mq_arb = Module(new Arbiter(new L2TlbInnerBundle, 2)) 149 mq_arb.io.in(0).valid := cache.io.resp.valid && !cache.io.resp.bits.hit && 150 (!cache.io.resp.bits.toFsm.l2Hit || cache.io.resp.bits.bypassed) && 151 !from_pre(cache.io.resp.bits.req_info.source) && 152 (cache.io.resp.bits.bypassed || cache.io.resp.bits.isFirst || !ptw.io.req.ready) 153 mq_arb.io.in(0).bits := cache.io.resp.bits.req_info 154 mq_arb.io.in(1) <> llptw.io.cache 155 missQueue.io.in <> mq_arb.io.out 156 missQueue.io.sfence := sfence_dup(6) 157 missQueue.io.csr := csr_dup(5) 158 159 blockmq.io.start := missQueue.io.out.fire 160 blockmq.io.enable := ptw.io.req.fire() 161 162 llptw.io.in.valid := cache.io.resp.valid && !cache.io.resp.bits.hit && cache.io.resp.bits.toFsm.l2Hit && !cache.io.resp.bits.bypassed 163 llptw.io.in.bits.req_info := cache.io.resp.bits.req_info 164 llptw.io.in.bits.ppn := cache.io.resp.bits.toFsm.ppn 165 llptw.io.sfence := sfence_dup(1) 166 llptw.io.csr := csr_dup(1) 167 168 cache.io.req.valid := arb2.io.out.valid 169 cache.io.req.bits.req_info.vpn := arb2.io.out.bits.vpn 170 cache.io.req.bits.req_info.source := arb2.io.out.bits.source 171 cache.io.req.bits.isFirst := arb2.io.chosen =/= InArbMissQueuePort.U 172 cache.io.req.bits.bypassed.map(_ := false.B) 173 cache.io.sfence := sfence_dup(2) 174 cache.io.csr := csr_dup(2) 175 cache.io.sfence_dup.zip(sfence_dup.drop(2).take(4)).map(s => s._1 := s._2) 176 cache.io.csr_dup.zip(csr_dup.drop(2).take(3)).map(c => c._1 := c._2) 177 cache.io.resp.ready := Mux(cache.io.resp.bits.hit, 178 outReady(cache.io.resp.bits.req_info.source, outArbCachePort), 179 Mux(cache.io.resp.bits.toFsm.l2Hit && !cache.io.resp.bits.bypassed, llptw.io.in.ready, 180 Mux(cache.io.resp.bits.bypassed || cache.io.resp.bits.isFirst, mq_arb.io.in(0).ready, mq_arb.io.in(0).ready || ptw.io.req.ready))) 181 182 // NOTE: missQueue req has higher priority 183 ptw.io.req.valid := cache.io.resp.valid && !cache.io.resp.bits.hit && !cache.io.resp.bits.toFsm.l2Hit && 184 !cache.io.resp.bits.bypassed && 185 !cache.io.resp.bits.isFirst 186 ptw.io.req.bits.req_info := cache.io.resp.bits.req_info 187 ptw.io.req.bits.l1Hit := cache.io.resp.bits.toFsm.l1Hit 188 ptw.io.req.bits.ppn := cache.io.resp.bits.toFsm.ppn 189 ptw.io.sfence := sfence_dup(7) 190 ptw.io.csr := csr_dup(6) 191 ptw.io.resp.ready := outReady(ptw.io.resp.bits.source, outArbFsmPort) 192 193 // mem req 194 def blockBytes_align(addr: UInt) = { 195 Cat(addr(PAddrBits - 1, log2Up(l2tlbParams.blockBytes)), 0.U(log2Up(l2tlbParams.blockBytes).W)) 196 } 197 def addr_low_from_vpn(vpn: UInt) = { 198 vpn(log2Ceil(l2tlbParams.blockBytes)-log2Ceil(XLEN/8)-1, 0) 199 } 200 def addr_low_from_paddr(paddr: UInt) = { 201 paddr(log2Up(l2tlbParams.blockBytes)-1, log2Up(XLEN/8)) 202 } 203 def from_missqueue(id: UInt) = { 204 (id =/= l2tlbParams.llptwsize.U) 205 } 206 val waiting_resp = RegInit(VecInit(Seq.fill(MemReqWidth)(false.B))) 207 val flush_latch = RegInit(VecInit(Seq.fill(MemReqWidth)(false.B))) 208 for (i <- waiting_resp.indices) { 209 assert(!flush_latch(i) || waiting_resp(i)) // when sfence_latch wait for mem resp, waiting_resp should be true 210 } 211 212 val llptw_out = llptw.io.out 213 val llptw_mem = llptw.io.mem 214 llptw_mem.req_mask := waiting_resp.take(l2tlbParams.llptwsize) 215 ptw.io.mem.mask := waiting_resp.last 216 217 val mem_arb = Module(new Arbiter(new L2TlbMemReqBundle(), 2)) 218 mem_arb.io.in(0) <> ptw.io.mem.req 219 mem_arb.io.in(1) <> llptw_mem.req 220 mem_arb.io.out.ready := mem.a.ready && !flush 221 222 // assert, should not send mem access at same addr for twice. 223 val last_resp_vpn = RegEnable(cache.io.refill.bits.req_info_dup(0).vpn, cache.io.refill.valid) 224 val last_resp_level = RegEnable(cache.io.refill.bits.level_dup(0), cache.io.refill.valid) 225 val last_resp_v = RegInit(false.B) 226 val last_has_invalid = !Cat(cache.io.refill.bits.ptes.asTypeOf(Vec(blockBits/XLEN, UInt(XLEN.W))).map(a => a(0))).andR || cache.io.refill.bits.sel_pte_dup(0).asTypeOf(new PteBundle).isAf() 227 when (cache.io.refill.valid) { last_resp_v := !last_has_invalid} 228 when (flush) { last_resp_v := false.B } 229 XSError(last_resp_v && cache.io.refill.valid && 230 (cache.io.refill.bits.req_info_dup(0).vpn === last_resp_vpn) && 231 (cache.io.refill.bits.level_dup(0) === last_resp_level), 232 "l2tlb should not access mem at same addr for twice") 233 // ATTENTION: this may wronngly assert when: a ptes is l2, last part is valid, 234 // but the current part is invalid, so one more mem access happened 235 // If this happened, remove the assert. 236 237 val req_addr_low = Reg(Vec(MemReqWidth, UInt((log2Up(l2tlbParams.blockBytes)-log2Up(XLEN/8)).W))) 238 239 when (llptw.io.in.fire()) { 240 // when enq miss queue, set the req_addr_low to receive the mem resp data part 241 req_addr_low(llptw_mem.enq_ptr) := addr_low_from_vpn(llptw.io.in.bits.req_info.vpn) 242 } 243 when (mem_arb.io.out.fire()) { 244 req_addr_low(mem_arb.io.out.bits.id) := addr_low_from_paddr(mem_arb.io.out.bits.addr) 245 waiting_resp(mem_arb.io.out.bits.id) := true.B 246 } 247 // mem read 248 val memRead = edge.Get( 249 fromSource = mem_arb.io.out.bits.id, 250 // toAddress = memAddr(log2Up(CacheLineSize / 2 / 8) - 1, 0), 251 toAddress = blockBytes_align(mem_arb.io.out.bits.addr), 252 lgSize = log2Up(l2tlbParams.blockBytes).U 253 )._2 254 mem.a.bits := memRead 255 mem.a.valid := mem_arb.io.out.valid && !flush 256 mem.a.bits.user.lift(ReqSourceKey).foreach(_ := MemReqSource.PTW.id.U) 257 mem.d.ready := true.B 258 // mem -> data buffer 259 val refill_data = Reg(Vec(blockBits / l1BusDataWidth, UInt(l1BusDataWidth.W))) 260 val refill_helper = edge.firstlastHelper(mem.d.bits, mem.d.fire()) 261 val mem_resp_done = refill_helper._3 262 val mem_resp_from_mq = from_missqueue(mem.d.bits.source) 263 when (mem.d.valid) { 264 assert(mem.d.bits.source <= l2tlbParams.llptwsize.U) 265 refill_data(refill_helper._4) := mem.d.bits.data 266 } 267 // refill_data_tmp is the wire fork of refill_data, but one cycle earlier 268 val refill_data_tmp = WireInit(refill_data) 269 refill_data_tmp(refill_helper._4) := mem.d.bits.data 270 271 // save only one pte for each id 272 // (miss queue may can't resp to tlb with low latency, it should have highest priority, but diffcult to design cache) 273 val resp_pte = VecInit((0 until MemReqWidth).map(i => 274 if (i == l2tlbParams.llptwsize) {RegEnable(get_part(refill_data_tmp, req_addr_low(i)), mem_resp_done && !mem_resp_from_mq) } 275 else { DataHoldBypass(get_part(refill_data, req_addr_low(i)), llptw_mem.buffer_it(i)) } 276 // llptw could not use refill_data_tmp, because enq bypass's result works at next cycle 277 )) 278 279 // save eight ptes for each id when sector tlb 280 // (miss queue may can't resp to tlb with low latency, it should have highest priority, but diffcult to design cache) 281 val resp_pte_sector = VecInit((0 until MemReqWidth).map(i => 282 if (i == l2tlbParams.llptwsize) {RegEnable(refill_data_tmp, mem_resp_done && !mem_resp_from_mq) } 283 else { DataHoldBypass(refill_data, llptw_mem.buffer_it(i)) } 284 // llptw could not use refill_data_tmp, because enq bypass's result works at next cycle 285 )) 286 287 // mem -> miss queue 288 llptw_mem.resp.valid := mem_resp_done && mem_resp_from_mq 289 llptw_mem.resp.bits.id := DataHoldBypass(mem.d.bits.source, mem.d.valid) 290 // mem -> ptw 291 ptw.io.mem.req.ready := mem.a.ready 292 ptw.io.mem.resp.valid := mem_resp_done && !mem_resp_from_mq 293 ptw.io.mem.resp.bits := resp_pte.last 294 // mem -> cache 295 val refill_from_mq = mem_resp_from_mq 296 val refill_level = Mux(refill_from_mq, 2.U, RegEnable(ptw.io.refill.level, init = 0.U, ptw.io.mem.req.fire())) 297 val refill_valid = mem_resp_done && !flush && !flush_latch(mem.d.bits.source) 298 299 cache.io.refill.valid := RegNext(refill_valid, false.B) 300 cache.io.refill.bits.ptes := refill_data.asUInt 301 cache.io.refill.bits.req_info_dup.map(_ := RegEnable(Mux(refill_from_mq, llptw_mem.refill, ptw.io.refill.req_info), refill_valid)) 302 cache.io.refill.bits.level_dup.map(_ := RegEnable(refill_level, refill_valid)) 303 cache.io.refill.bits.levelOH(refill_level, refill_valid) 304 cache.io.refill.bits.sel_pte_dup.map(_ := RegNext(sel_data(refill_data_tmp.asUInt, req_addr_low(mem.d.bits.source)))) 305 306 if (env.EnableDifftest) { 307 val difftest_ptw_addr = RegInit(VecInit(Seq.fill(MemReqWidth)(0.U(PAddrBits.W)))) 308 when (mem.a.valid) { 309 difftest_ptw_addr(mem.a.bits.source) := mem.a.bits.address 310 } 311 312 val difftest = DifftestModule(new DiffRefillEvent) 313 difftest.clock := clock 314 difftest.coreid := p(XSCoreParamsKey).HartId.asUInt 315 difftest.index := 2.U 316 difftest.valid := cache.io.refill.valid 317 difftest.addr := difftest_ptw_addr(RegNext(mem.d.bits.source)) 318 difftest.data := refill_data.asTypeOf(difftest.data) 319 } 320 321 if (env.EnableDifftest) { 322 for (i <- 0 until PtwWidth) { 323 val difftest = DifftestModule(new DiffL2TLBEvent) 324 difftest.clock := clock 325 difftest.coreid := p(XSCoreParamsKey).HartId.asUInt 326 difftest.valid := io.tlb(i).resp.fire && !io.tlb(i).resp.bits.af 327 difftest.index := i.U 328 difftest.satp := io.csr.tlb.satp.ppn 329 difftest.vpn := Cat(io.tlb(i).resp.bits.entry.tag, 0.U(sectortlbwidth.W)) 330 for (j <- 0 until tlbcontiguous) { 331 difftest.ppn(j) := Cat(io.tlb(i).resp.bits.entry.ppn, io.tlb(i).resp.bits.ppn_low(j)) 332 difftest.valididx(j) := io.tlb(i).resp.bits.valididx(j) 333 } 334 difftest.perm := io.tlb(i).resp.bits.entry.perm.getOrElse(0.U.asTypeOf(new PtePermBundle)).asUInt 335 difftest.level := io.tlb(i).resp.bits.entry.level.getOrElse(0.U.asUInt) 336 difftest.pf := io.tlb(i).resp.bits.pf 337 } 338 } 339 340 // pmp 341 pmp_check(0).req <> ptw.io.pmp.req 342 ptw.io.pmp.resp <> pmp_check(0).resp 343 pmp_check(1).req <> llptw.io.pmp.req 344 llptw.io.pmp.resp <> pmp_check(1).resp 345 346 llptw_out.ready := outReady(llptw_out.bits.req_info.source, outArbMqPort) 347 348 // Timing: Maybe need to do some optimization or even add one more cycle 349 for (i <- 0 until PtwWidth) { 350 mergeArb(i).in(outArbCachePort).valid := cache.io.resp.valid && cache.io.resp.bits.hit && cache.io.resp.bits.req_info.source===i.U 351 mergeArb(i).in(outArbCachePort).bits := cache.io.resp.bits.toTlb 352 mergeArb(i).in(outArbFsmPort).valid := ptw.io.resp.valid && ptw.io.resp.bits.source===i.U 353 mergeArb(i).in(outArbFsmPort).bits := ptw.io.resp.bits.resp 354 mergeArb(i).in(outArbMqPort).valid := llptw_out.valid && llptw_out.bits.req_info.source===i.U 355 mergeArb(i).in(outArbMqPort).bits := contiguous_pte_to_merge_ptwResp(resp_pte_sector(llptw_out.bits.id).asUInt, llptw_out.bits.req_info.vpn, llptw_out.bits.af, true) 356 mergeArb(i).out.ready := outArb(i).in(0).ready 357 } 358 359 for (i <- 0 until PtwWidth) { 360 outArb(i).in(0).valid := mergeArb(i).out.valid 361 outArb(i).in(0).bits := merge_ptwResp_to_sector_ptwResp(mergeArb(i).out.bits) 362 } 363 364 // io.tlb.map(_.resp) <> outArb.map(_.out) 365 io.tlb.map(_.resp).zip(outArb.map(_.out)).map{ 366 case (resp, out) => resp <> out 367 } 368 369 // sfence 370 when (flush) { 371 for (i <- 0 until MemReqWidth) { 372 when (waiting_resp(i)) { 373 flush_latch(i) := true.B 374 } 375 } 376 } 377 // mem -> control signal 378 // waiting_resp and sfence_latch will be reset when mem_resp_done 379 when (mem_resp_done) { 380 waiting_resp(mem.d.bits.source) := false.B 381 flush_latch(mem.d.bits.source) := false.B 382 } 383 384 def block_decoupled[T <: Data](source: DecoupledIO[T], sink: DecoupledIO[T], block_signal: Bool) = { 385 sink.valid := source.valid && !block_signal 386 source.ready := sink.ready && !block_signal 387 sink.bits := source.bits 388 } 389 390 def get_part(data: Vec[UInt], index: UInt): UInt = { 391 val inner_data = data.asTypeOf(Vec(data.getWidth / XLEN, UInt(XLEN.W))) 392 inner_data(index) 393 } 394 395 def pte_to_ptwResp(pte: UInt, vpn: UInt, af: Bool, af_first: Boolean) : PtwResp = { 396 val pte_in = pte.asTypeOf(new PteBundle()) 397 val ptw_resp = Wire(new PtwResp()) 398 ptw_resp.entry.ppn := pte_in.ppn 399 ptw_resp.entry.level.map(_ := 2.U) 400 ptw_resp.entry.perm.map(_ := pte_in.getPerm()) 401 ptw_resp.entry.tag := vpn 402 ptw_resp.pf := (if (af_first) !af else true.B) && pte_in.isPf(2.U) 403 ptw_resp.af := (if (!af_first) pte_in.isPf(2.U) else true.B) && (af || pte_in.isAf()) 404 ptw_resp.entry.v := !ptw_resp.pf 405 ptw_resp.entry.prefetch := DontCare 406 ptw_resp.entry.asid := satp.asid 407 ptw_resp 408 } 409 410 // not_super means that this is a normal page 411 // valididx(i) will be all true when super page to be convenient for l1 tlb matching 412 def contiguous_pte_to_merge_ptwResp(pte: UInt, vpn: UInt, af: Bool, af_first: Boolean, not_super: Boolean = true) : PtwMergeResp = { 413 assert(tlbcontiguous == 8, "Only support tlbcontiguous = 8!") 414 val ptw_merge_resp = Wire(new PtwMergeResp()) 415 for (i <- 0 until tlbcontiguous) { 416 val pte_in = pte(64 * i + 63, 64 * i).asTypeOf(new PteBundle()) 417 val ptw_resp = Wire(new PtwMergeEntry(tagLen = sectorvpnLen, hasPerm = true, hasLevel = true)) 418 ptw_resp.ppn := pte_in.ppn(ppnLen - 1, sectortlbwidth) 419 ptw_resp.ppn_low := pte_in.ppn(sectortlbwidth - 1, 0) 420 ptw_resp.level.map(_ := 2.U) 421 ptw_resp.perm.map(_ := pte_in.getPerm()) 422 ptw_resp.tag := vpn(vpnLen - 1, sectortlbwidth) 423 ptw_resp.pf := (if (af_first) !af else true.B) && pte_in.isPf(2.U) 424 ptw_resp.af := (if (!af_first) pte_in.isPf(2.U) else true.B) && (af || pte_in.isAf()) 425 ptw_resp.v := !ptw_resp.pf 426 ptw_resp.prefetch := DontCare 427 ptw_resp.asid := satp.asid 428 ptw_merge_resp.entry(i) := ptw_resp 429 } 430 ptw_merge_resp.pteidx := UIntToOH(vpn(sectortlbwidth - 1, 0)).asBools 431 ptw_merge_resp.not_super := not_super.B 432 ptw_merge_resp 433 } 434 435 def merge_ptwResp_to_sector_ptwResp(pte: PtwMergeResp) : PtwSectorResp = { 436 assert(tlbcontiguous == 8, "Only support tlbcontiguous = 8!") 437 val ptw_sector_resp = Wire(new PtwSectorResp) 438 ptw_sector_resp.entry.tag := pte.entry(OHToUInt(pte.pteidx)).tag 439 ptw_sector_resp.entry.asid := pte.entry(OHToUInt(pte.pteidx)).asid 440 ptw_sector_resp.entry.ppn := pte.entry(OHToUInt(pte.pteidx)).ppn 441 ptw_sector_resp.entry.perm.map(_ := pte.entry(OHToUInt(pte.pteidx)).perm.getOrElse(0.U.asTypeOf(new PtePermBundle))) 442 ptw_sector_resp.entry.level.map(_ := pte.entry(OHToUInt(pte.pteidx)).level.getOrElse(0.U(2.W))) 443 ptw_sector_resp.entry.prefetch := pte.entry(OHToUInt(pte.pteidx)).prefetch 444 ptw_sector_resp.entry.v := pte.entry(OHToUInt(pte.pteidx)).v 445 ptw_sector_resp.af := pte.entry(OHToUInt(pte.pteidx)).af 446 ptw_sector_resp.pf := pte.entry(OHToUInt(pte.pteidx)).pf 447 ptw_sector_resp.addr_low := OHToUInt(pte.pteidx) 448 ptw_sector_resp.pteidx := pte.pteidx 449 for (i <- 0 until tlbcontiguous) { 450 val ppn_equal = pte.entry(i).ppn === pte.entry(OHToUInt(pte.pteidx)).ppn 451 val perm_equal = pte.entry(i).perm.getOrElse(0.U.asTypeOf(new PtePermBundle)).asUInt === pte.entry(OHToUInt(pte.pteidx)).perm.getOrElse(0.U.asTypeOf(new PtePermBundle)).asUInt 452 val v_equal = pte.entry(i).v === pte.entry(OHToUInt(pte.pteidx)).v 453 val af_equal = pte.entry(i).af === pte.entry(OHToUInt(pte.pteidx)).af 454 val pf_equal = pte.entry(i).pf === pte.entry(OHToUInt(pte.pteidx)).pf 455 ptw_sector_resp.valididx(i) := (ppn_equal && perm_equal && v_equal && af_equal && pf_equal) || !pte.not_super 456 ptw_sector_resp.ppn_low(i) := pte.entry(i).ppn_low 457 } 458 ptw_sector_resp.valididx(OHToUInt(pte.pteidx)) := true.B 459 ptw_sector_resp 460 } 461 462 def outReady(source: UInt, port: Int): Bool = { 463 MuxLookup(source, true.B, 464 (0 until PtwWidth).map(i => i.U -> mergeArb(i).in(port).ready)) 465 } 466 467 // debug info 468 for (i <- 0 until PtwWidth) { 469 XSDebug(p"[io.tlb(${i.U})] ${io.tlb(i)}\n") 470 } 471 XSDebug(p"[sfence] ${io.sfence}\n") 472 XSDebug(p"[io.csr.tlb] ${io.csr.tlb}\n") 473 474 for (i <- 0 until PtwWidth) { 475 XSPerfAccumulate(s"req_count${i}", io.tlb(i).req(0).fire()) 476 XSPerfAccumulate(s"req_blocked_count_${i}", io.tlb(i).req(0).valid && !io.tlb(i).req(0).ready) 477 } 478 XSPerfAccumulate(s"req_blocked_by_mq", arb1.io.out.valid && missQueue.io.out.valid) 479 for (i <- 0 until (MemReqWidth + 1)) { 480 XSPerfAccumulate(s"mem_req_util${i}", PopCount(waiting_resp) === i.U) 481 } 482 XSPerfAccumulate("mem_cycle", PopCount(waiting_resp) =/= 0.U) 483 XSPerfAccumulate("mem_count", mem.a.fire()) 484 for (i <- 0 until PtwWidth) { 485 XSPerfAccumulate(s"llptw_ppn_af${i}", mergeArb(i).in(outArbMqPort).valid && mergeArb(i).in(outArbMqPort).bits.entry(OHToUInt(mergeArb(i).in(outArbMqPort).bits.pteidx)).af && !llptw_out.bits.af) 486 XSPerfAccumulate(s"access_fault${i}", io.tlb(i).resp.fire && io.tlb(i).resp.bits.af) 487 } 488 489 // print configs 490 println(s"${l2tlbParams.name}: a ptw, a llptw with size ${l2tlbParams.llptwsize}, miss queue size ${MissQueueSize} l1:${l2tlbParams.l1Size} fa l2: nSets ${l2tlbParams.l2nSets} nWays ${l2tlbParams.l2nWays} l3: ${l2tlbParams.l3nSets} nWays ${l2tlbParams.l3nWays} blockBytes:${l2tlbParams.blockBytes}") 491 492 // time out assert 493 for (i <- 0 until MemReqWidth) { 494 TimeOutAssert(waiting_resp(i), timeOutThreshold, s"ptw mem resp time out wait_resp${i}") 495 TimeOutAssert(flush_latch(i), timeOutThreshold, s"ptw mem resp time out flush_latch${i}") 496 } 497 498 499 val perfEvents = Seq(llptw, cache, ptw).flatMap(_.getPerfEvents) 500 generatePerfEvent() 501 502 val isWriteL1TlbTable = WireInit(Constantin.createRecord("isWriteL1TlbTable" + p(XSCoreParamsKey).HartId.toString)) 503 val L1TlbTable = ChiselDB.createTable("L1Tlb_hart" + p(XSCoreParamsKey).HartId.toString, new L1TlbDB) 504 val ITlbReqDB, DTlbReqDB, ITlbRespDB, DTlbRespDB = Wire(new L1TlbDB) 505 ITlbReqDB.vpn := io.tlb(0).req(0).bits.vpn 506 DTlbReqDB.vpn := io.tlb(1).req(0).bits.vpn 507 ITlbRespDB.vpn := io.tlb(0).resp.bits.entry.tag 508 DTlbRespDB.vpn := io.tlb(1).resp.bits.entry.tag 509 L1TlbTable.log(ITlbReqDB, isWriteL1TlbTable.orR && io.tlb(0).req(0).fire, "ITlbReq", clock, reset) 510 L1TlbTable.log(DTlbReqDB, isWriteL1TlbTable.orR && io.tlb(1).req(0).fire, "DTlbReq", clock, reset) 511 L1TlbTable.log(ITlbRespDB, isWriteL1TlbTable.orR && io.tlb(0).resp.fire, "ITlbResp", clock, reset) 512 L1TlbTable.log(DTlbRespDB, isWriteL1TlbTable.orR && io.tlb(1).resp.fire, "DTlbResp", clock, reset) 513 514 val isWritePageCacheTable = WireInit(Constantin.createRecord("isWritePageCacheTable" + p(XSCoreParamsKey).HartId.toString)) 515 val PageCacheTable = ChiselDB.createTable("PageCache_hart" + p(XSCoreParamsKey).HartId.toString, new PageCacheDB) 516 val PageCacheDB = Wire(new PageCacheDB) 517 PageCacheDB.vpn := Cat(cache.io.resp.bits.toTlb.entry(0).tag, OHToUInt(cache.io.resp.bits.toTlb.pteidx)) 518 PageCacheDB.source := cache.io.resp.bits.req_info.source 519 PageCacheDB.bypassed := cache.io.resp.bits.bypassed 520 PageCacheDB.is_first := cache.io.resp.bits.isFirst 521 PageCacheDB.prefetched := cache.io.resp.bits.toTlb.entry(0).prefetch 522 PageCacheDB.prefetch := cache.io.resp.bits.prefetch 523 PageCacheDB.l2Hit := cache.io.resp.bits.toFsm.l2Hit 524 PageCacheDB.l1Hit := cache.io.resp.bits.toFsm.l1Hit 525 PageCacheDB.hit := cache.io.resp.bits.hit 526 PageCacheTable.log(PageCacheDB, isWritePageCacheTable.orR && cache.io.resp.fire, "PageCache", clock, reset) 527 528 val isWritePTWTable = WireInit(Constantin.createRecord("isWritePTWTable" + p(XSCoreParamsKey).HartId.toString)) 529 val PTWTable = ChiselDB.createTable("PTW_hart" + p(XSCoreParamsKey).HartId.toString, new PTWDB) 530 val PTWReqDB, PTWRespDB, LLPTWReqDB, LLPTWRespDB = Wire(new PTWDB) 531 PTWReqDB.vpn := ptw.io.req.bits.req_info.vpn 532 PTWReqDB.source := ptw.io.req.bits.req_info.source 533 PTWRespDB.vpn := ptw.io.refill.req_info.vpn 534 PTWRespDB.source := ptw.io.refill.req_info.source 535 LLPTWReqDB.vpn := llptw.io.in.bits.req_info.vpn 536 LLPTWReqDB.source := llptw.io.in.bits.req_info.source 537 LLPTWRespDB.vpn := llptw.io.mem.refill.vpn 538 LLPTWRespDB.source := llptw.io.mem.refill.source 539 PTWTable.log(PTWReqDB, isWritePTWTable.orR && ptw.io.req.fire, "PTWReq", clock, reset) 540 PTWTable.log(PTWRespDB, isWritePTWTable.orR && ptw.io.mem.resp.fire, "PTWResp", clock, reset) 541 PTWTable.log(LLPTWReqDB, isWritePTWTable.orR && llptw.io.in.fire, "LLPTWReq", clock, reset) 542 PTWTable.log(LLPTWRespDB, isWritePTWTable.orR && llptw.io.mem.resp.fire, "LLPTWResp", clock, reset) 543 544 val isWriteL2TlbMissQueueTable = WireInit(Constantin.createRecord("isWriteL2TlbMissQueueTable" + p(XSCoreParamsKey).HartId.toString)) 545 val L2TlbMissQueueTable = ChiselDB.createTable("L2TlbMissQueue_hart" + p(XSCoreParamsKey).HartId.toString, new L2TlbMissQueueDB) 546 val L2TlbMissQueueInDB, L2TlbMissQueueOutDB = Wire(new L2TlbMissQueueDB) 547 L2TlbMissQueueInDB.vpn := missQueue.io.in.bits.vpn 548 L2TlbMissQueueOutDB.vpn := missQueue.io.out.bits.vpn 549 L2TlbMissQueueTable.log(L2TlbMissQueueInDB, isWriteL2TlbMissQueueTable.orR && missQueue.io.in.fire, "L2TlbMissQueueIn", clock, reset) 550 L2TlbMissQueueTable.log(L2TlbMissQueueOutDB, isWriteL2TlbMissQueueTable.orR && missQueue.io.out.fire, "L2TlbMissQueueOut", clock, reset) 551} 552 553/** BlockHelper, block missqueue, not to send too many req to cache 554 * Parameter: 555 * enable: enable BlockHelper, mq should not send too many reqs 556 * start: when miss queue out fire and need, block miss queue's out 557 * block: block miss queue's out 558 * latency: last missqueue out's cache access latency 559 */ 560class BlockHelper(latency: Int)(implicit p: Parameters) extends XSModule { 561 val io = IO(new Bundle { 562 val enable = Input(Bool()) 563 val start = Input(Bool()) 564 val block = Output(Bool()) 565 }) 566 567 val count = RegInit(0.U(log2Ceil(latency).W)) 568 val valid = RegInit(false.B) 569 val work = RegInit(true.B) 570 571 io.block := valid 572 573 when (io.start && work) { valid := true.B } 574 when (valid) { count := count + 1.U } 575 when (count === (latency.U) || io.enable) { 576 valid := false.B 577 work := io.enable 578 count := 0.U 579 } 580} 581 582class PTEHelper() extends ExtModule { 583 val clock = IO(Input(Clock())) 584 val enable = IO(Input(Bool())) 585 val satp = IO(Input(UInt(64.W))) 586 val vpn = IO(Input(UInt(64.W))) 587 val pte = IO(Output(UInt(64.W))) 588 val level = IO(Output(UInt(8.W))) 589 val pf = IO(Output(UInt(8.W))) 590} 591 592class PTWDelayN[T <: Data](gen: T, n: Int, flush: Bool) extends Module { 593 val io = IO(new Bundle() { 594 val in = Input(gen) 595 val out = Output(gen) 596 val ptwflush = Input(flush.cloneType) 597 }) 598 val out = RegInit(VecInit(Seq.fill(n)(0.U.asTypeOf(gen)))) 599 val t = RegInit(VecInit(Seq.fill(n)(0.U.asTypeOf(gen)))) 600 out(0) := io.in 601 if (n == 1) { 602 io.out := out(0) 603 } else { 604 when (io.ptwflush) { 605 for (i <- 0 until n) { 606 t(i) := 0.U.asTypeOf(gen) 607 out(i) := 0.U.asTypeOf(gen) 608 } 609 io.out := 0.U.asTypeOf(gen) 610 } .otherwise { 611 for (i <- 1 until n) { 612 t(i-1) := out(i-1) 613 out(i) := t(i-1) 614 } 615 io.out := out(n-1) 616 } 617 } 618} 619 620object PTWDelayN { 621 def apply[T <: Data](in: T, n: Int, flush: Bool): T = { 622 val delay = Module(new PTWDelayN(in.cloneType, n, flush)) 623 delay.io.in := in 624 delay.io.ptwflush := flush 625 delay.io.out 626 } 627} 628 629class FakePTW()(implicit p: Parameters) extends XSModule with HasPtwConst { 630 val io = IO(new L2TLBIO) 631 val flush = VecInit(Seq.fill(PtwWidth)(false.B)) 632 flush(0) := DelayN(io.sfence.valid || io.csr.tlb.satp.changed, itlbParams.fenceDelay) 633 flush(1) := DelayN(io.sfence.valid || io.csr.tlb.satp.changed, ldtlbParams.fenceDelay) 634 for (i <- 0 until PtwWidth) { 635 val helper = Module(new PTEHelper()) 636 helper.clock := clock 637 helper.satp := io.csr.tlb.satp.ppn 638 639 if (coreParams.softPTWDelay == 1) { 640 helper.enable := io.tlb(i).req(0).fire 641 helper.vpn := io.tlb(i).req(0).bits.vpn 642 } else { 643 helper.enable := PTWDelayN(io.tlb(i).req(0).fire, coreParams.softPTWDelay - 1, flush(i)) 644 helper.vpn := PTWDelayN(io.tlb(i).req(0).bits.vpn, coreParams.softPTWDelay - 1, flush(i)) 645 } 646 647 val pte = helper.pte.asTypeOf(new PteBundle) 648 val level = helper.level 649 val pf = helper.pf 650 val empty = RegInit(true.B) 651 when (io.tlb(i).req(0).fire) { 652 empty := false.B 653 } .elsewhen (io.tlb(i).resp.fire || flush(i)) { 654 empty := true.B 655 } 656 657 io.tlb(i).req(0).ready := empty || io.tlb(i).resp.fire 658 io.tlb(i).resp.valid := PTWDelayN(io.tlb(i).req(0).fire, coreParams.softPTWDelay, flush(i)) 659 assert(!io.tlb(i).resp.valid || io.tlb(i).resp.ready) 660 io.tlb(i).resp.bits.entry.tag := PTWDelayN(io.tlb(i).req(0).bits.vpn, coreParams.softPTWDelay, flush(i)) 661 io.tlb(i).resp.bits.entry.ppn := pte.ppn 662 io.tlb(i).resp.bits.entry.perm.map(_ := pte.getPerm()) 663 io.tlb(i).resp.bits.entry.level.map(_ := level) 664 io.tlb(i).resp.bits.pf := pf 665 io.tlb(i).resp.bits.af := DontCare // TODO: implement it 666 io.tlb(i).resp.bits.entry.v := !pf 667 io.tlb(i).resp.bits.entry.prefetch := DontCare 668 io.tlb(i).resp.bits.entry.asid := io.csr.tlb.satp.asid 669 } 670} 671 672class L2TLBWrapper()(implicit p: Parameters) extends LazyModule with HasXSParameter { 673 val useSoftPTW = coreParams.softPTW 674 val node = if (!useSoftPTW) TLIdentityNode() else null 675 val ptw = if (!useSoftPTW) LazyModule(new L2TLB()) else null 676 if (!useSoftPTW) { 677 node := ptw.node 678 } 679 680 lazy val module = new LazyModuleImp(this) with HasPerfEvents { 681 val io = IO(new L2TLBIO) 682 val perfEvents = if (useSoftPTW) { 683 val fake_ptw = Module(new FakePTW()) 684 io <> fake_ptw.io 685 Seq() 686 } 687 else { 688 io <> ptw.module.io 689 ptw.module.getPerfEvents 690 } 691 generatePerfEvent() 692 } 693} 694