1/*************************************************************************************** 2* Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences 3* Copyright (c) 2020-2021 Peng Cheng Laboratory 4* 5* XiangShan is licensed under Mulan PSL v2. 6* You can use this software according to the terms and conditions of the Mulan PSL v2. 7* You may obtain a copy of Mulan PSL v2 at: 8* http://license.coscl.org.cn/MulanPSL2 9* 10* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, 11* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, 12* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. 13* 14* See the Mulan PSL v2 for more details. 15***************************************************************************************/ 16 17package xiangshan.cache.mmu 18 19import chipsalliance.rocketchip.config.Parameters 20import chisel3._ 21import chisel3.util._ 22import xiangshan._ 23import xiangshan.cache.{HasDCacheParameters, MemoryOpConstants} 24import utils._ 25import utility._ 26import freechips.rocketchip.diplomacy.{LazyModule, LazyModuleImp} 27import freechips.rocketchip.tilelink._ 28 29class PTWReapterIO(Width: Int)(implicit p: Parameters) extends MMUIOBaseBundle { 30 val tlb = Flipped(new TlbPtwIO(Width)) 31 val ptw = new TlbPtwIO 32 33 def apply(tlb: TlbPtwIO, ptw: TlbPtwIO, sfence: SfenceBundle, csr: TlbCsrBundle): Unit = { 34 this.tlb <> tlb 35 this.ptw <> ptw 36 this.sfence <> sfence 37 this.csr <> csr 38 } 39 40 def apply(tlb: TlbPtwIO, sfence: SfenceBundle, csr: TlbCsrBundle): Unit = { 41 this.tlb <> tlb 42 this.sfence <> sfence 43 this.csr <> csr 44 } 45 46} 47 48class PTWRepeater(Width: Int = 1, FenceDelay: Int)(implicit p: Parameters) extends XSModule with HasPtwConst { 49 val io = IO(new PTWReapterIO(Width)) 50 51 val req_in = if (Width == 1) { 52 io.tlb.req(0) 53 } else { 54 val arb = Module(new RRArbiter(io.tlb.req(0).bits.cloneType, Width)) 55 arb.io.in <> io.tlb.req 56 arb.io.out 57 } 58 val (tlb, ptw, flush) = (io.tlb, io.ptw, DelayN(io.sfence.valid || io.csr.satp.changed, FenceDelay)) 59 val req = RegEnable(req_in.bits, req_in.fire()) 60 val resp = RegEnable(ptw.resp.bits, ptw.resp.fire()) 61 val haveOne = BoolStopWatch(req_in.fire(), tlb.resp.fire() || flush) 62 val sent = BoolStopWatch(ptw.req(0).fire(), req_in.fire() || flush) 63 val recv = BoolStopWatch(ptw.resp.fire() && haveOne, req_in.fire() || flush) 64 65 req_in.ready := !haveOne 66 ptw.req(0).valid := haveOne && !sent 67 ptw.req(0).bits := req 68 69 tlb.resp.bits := resp 70 tlb.resp.valid := haveOne && recv 71 ptw.resp.ready := !recv 72 73 XSPerfAccumulate("req_count", ptw.req(0).fire()) 74 XSPerfAccumulate("tlb_req_cycle", BoolStopWatch(req_in.fire(), tlb.resp.fire() || flush)) 75 XSPerfAccumulate("ptw_req_cycle", BoolStopWatch(ptw.req(0).fire(), ptw.resp.fire() || flush)) 76 77 XSDebug(haveOne, p"haveOne:${haveOne} sent:${sent} recv:${recv} sfence:${flush} req:${req} resp:${resp}") 78 XSDebug(req_in.valid || io.tlb.resp.valid, p"tlb: ${tlb}\n") 79 XSDebug(io.ptw.req(0).valid || io.ptw.resp.valid, p"ptw: ${ptw}\n") 80 assert(!RegNext(recv && io.ptw.resp.valid, init = false.B), "re-receive ptw.resp") 81 XSError(io.ptw.req(0).valid && io.ptw.resp.valid && !flush, "ptw repeater recv resp when sending") 82 XSError(io.ptw.resp.valid && (req.vpn =/= io.ptw.resp.bits.entry.tag), "ptw repeater recv resp with wrong tag") 83 XSError(io.ptw.resp.valid && !io.ptw.resp.ready, "ptw repeater's ptw resp back, but not ready") 84 TimeOutAssert(sent && !recv, timeOutThreshold, "Repeater doesn't recv resp in time") 85} 86 87/* dtlb 88 * 89 */ 90 91class PTWRepeaterNB(Width: Int = 1, passReady: Boolean = false, FenceDelay: Int)(implicit p: Parameters) extends XSModule with HasPtwConst { 92 val io = IO(new PTWReapterIO(Width)) 93 94 val req_in = if (Width == 1) { 95 io.tlb.req(0) 96 } else { 97 val arb = Module(new RRArbiter(io.tlb.req(0).bits.cloneType, Width)) 98 arb.io.in <> io.tlb.req 99 arb.io.out 100 } 101 val (tlb, ptw, flush) = (io.tlb, io.ptw, DelayN(io.sfence.valid || io.csr.satp.changed, FenceDelay)) 102 /* sent: tlb -> repeater -> ptw 103 * recv: ptw -> repeater -> tlb 104 * different from PTWRepeater 105 */ 106 107 // tlb -> repeater -> ptw 108 val req = RegEnable(req_in.bits, req_in.fire()) 109 val sent = BoolStopWatch(req_in.fire(), ptw.req(0).fire() || flush) 110 req_in.ready := !sent || { if (passReady) ptw.req(0).ready else false.B } 111 ptw.req(0).valid := sent 112 ptw.req(0).bits := req 113 114 // ptw -> repeater -> tlb 115 val resp = RegEnable(ptw.resp.bits, ptw.resp.fire()) 116 val recv = BoolStopWatch(ptw.resp.fire(), tlb.resp.fire() || flush) 117 ptw.resp.ready := !recv || { if (passReady) tlb.resp.ready else false.B } 118 tlb.resp.valid := recv 119 tlb.resp.bits := resp 120 121 XSPerfAccumulate("req", req_in.fire()) 122 XSPerfAccumulate("resp", tlb.resp.fire()) 123 if (!passReady) { 124 XSPerfAccumulate("req_blank", req_in.valid && sent && ptw.req(0).ready) 125 XSPerfAccumulate("resp_blank", ptw.resp.valid && recv && tlb.resp.ready) 126 XSPerfAccumulate("req_blank_ignore_ready", req_in.valid && sent) 127 XSPerfAccumulate("resp_blank_ignore_ready", ptw.resp.valid && recv) 128 } 129 XSDebug(req_in.valid || io.tlb.resp.valid, p"tlb: ${tlb}\n") 130 XSDebug(io.ptw.req(0).valid || io.ptw.resp.valid, p"ptw: ${ptw}\n") 131} 132 133class PTWFilterIO(Width: Int)(implicit p: Parameters) extends MMUIOBaseBundle { 134 val tlb = Flipped(new VectorTlbPtwIO(Width)) 135 val ptw = new TlbPtwIO() 136 val rob_head_miss_in_tlb = Output(Bool()) 137 138 def apply(tlb: VectorTlbPtwIO, ptw: TlbPtwIO, sfence: SfenceBundle, csr: TlbCsrBundle): Unit = { 139 this.tlb <> tlb 140 this.ptw <> ptw 141 this.sfence <> sfence 142 this.csr <> csr 143 } 144 145 def apply(tlb: VectorTlbPtwIO, sfence: SfenceBundle, csr: TlbCsrBundle): Unit = { 146 this.tlb <> tlb 147 this.sfence <> sfence 148 this.csr <> csr 149 } 150 151} 152 153class PTWFilter(Width: Int, Size: Int, FenceDelay: Int)(implicit p: Parameters) extends XSModule with HasPtwConst { 154 require(Size >= Width) 155 156 val io = IO(new PTWFilterIO(Width)) 157 158 val v = RegInit(VecInit(Seq.fill(Size)(false.B))) 159 val ports = Reg(Vec(Size, Vec(Width, Bool()))) // record which port(s) the entry come from, may not able to cover all the ports 160 val vpn = Reg(Vec(Size, UInt(vpnLen.W))) 161 val memidx = Reg(Vec(Size, new MemBlockidxBundle)) 162 val enqPtr = RegInit(0.U(log2Up(Size).W)) // Enq 163 val issPtr = RegInit(0.U(log2Up(Size).W)) // Iss to Ptw 164 val deqPtr = RegInit(0.U(log2Up(Size).W)) // Deq 165 val mayFullDeq = RegInit(false.B) 166 val mayFullIss = RegInit(false.B) 167 val counter = RegInit(0.U(log2Up(Size+1).W)) 168 169 val flush = DelayN(io.sfence.valid || io.csr.satp.changed, FenceDelay) 170 val tlb_req = WireInit(io.tlb.req) // NOTE: tlb_req is not io.tlb.req, see below codes, just use cloneType 171 tlb_req.suggestName("tlb_req") 172 173 val inflight_counter = RegInit(0.U(log2Up(Size + 1).W)) 174 val inflight_full = inflight_counter === Size.U 175 when (io.ptw.req(0).fire() =/= io.ptw.resp.fire()) { 176 inflight_counter := Mux(io.ptw.req(0).fire(), inflight_counter + 1.U, inflight_counter - 1.U) 177 } 178 179 val canEnqueue = Wire(Bool()) // NOTE: actually enqueue 180 val ptwResp = RegEnable(io.ptw.resp.bits, io.ptw.resp.fire()) 181 val ptwResp_OldMatchVec = vpn.zip(v).map{ case (pi, vi) => 182 vi && io.ptw.resp.bits.hit(pi, io.csr.satp.asid, true, true)} 183 val ptwResp_valid = RegNext(io.ptw.resp.fire() && Cat(ptwResp_OldMatchVec).orR, init = false.B) 184 // May send repeated requests to L2 tlb with same vpn(26, 3) when sector tlb 185 val oldMatchVec_early = io.tlb.req.map(a => vpn.zip(v).map{ case (pi, vi) => vi && pi === a.bits.vpn}) 186 val lastReqMatchVec_early = io.tlb.req.map(a => tlb_req.map{ b => b.valid && b.bits.vpn === a.bits.vpn && canEnqueue}) 187 val newMatchVec_early = io.tlb.req.map(a => io.tlb.req.map(b => a.bits.vpn === b.bits.vpn)) 188 189 (0 until Width) foreach { i => 190 tlb_req(i).valid := RegNext(io.tlb.req(i).valid && 191 !(ptwResp_valid && ptwResp.hit(io.tlb.req(i).bits.vpn, 0.U, true, true)) && 192 !Cat(lastReqMatchVec_early(i)).orR, 193 init = false.B) 194 tlb_req(i).bits := RegEnable(io.tlb.req(i).bits, io.tlb.req(i).valid) 195 } 196 197 val oldMatchVec = oldMatchVec_early.map(a => RegNext(Cat(a).orR)) 198 val newMatchVec = (0 until Width).map(i => (0 until Width).map(j => 199 RegNext(newMatchVec_early(i)(j)) && tlb_req(j).valid 200 )) 201 val ptwResp_newMatchVec = tlb_req.map(a => 202 ptwResp_valid && ptwResp.hit(a.bits.vpn, 0.U, allType = true, true)) 203 204 val oldMatchVec2 = (0 until Width).map(i => oldMatchVec_early(i).map(RegNext(_)).map(_ & tlb_req(i).valid)) 205 val update_ports = v.indices.map(i => oldMatchVec2.map(j => j(i))) 206 val ports_init = (0 until Width).map(i => (1 << i).U(Width.W)) 207 val filter_ports = (0 until Width).map(i => ParallelMux(newMatchVec(i).zip(ports_init).drop(i))) 208 val resp_vector = RegEnable(ParallelMux(ptwResp_OldMatchVec zip ports), io.ptw.resp.fire()) 209 210 def canMerge(index: Int) : Bool = { 211 ptwResp_newMatchVec(index) || oldMatchVec(index) || 212 Cat(newMatchVec(index).take(index)).orR 213 } 214 215 def filter_req() = { 216 val reqs = tlb_req.indices.map{ i => 217 val req = Wire(ValidIO(new PtwReqwithMemIdx())) 218 val merge = canMerge(i) 219 req.bits := tlb_req(i).bits 220 req.valid := !merge && tlb_req(i).valid 221 req 222 } 223 reqs 224 } 225 226 val reqs = filter_req() 227 val req_ports = filter_ports 228 val isFull = enqPtr === deqPtr && mayFullDeq 229 val isEmptyDeq = enqPtr === deqPtr && !mayFullDeq 230 val isEmptyIss = enqPtr === issPtr && !mayFullIss 231 val accumEnqNum = (0 until Width).map(i => PopCount(reqs.take(i).map(_.valid))) 232 val enqPtrVecInit = VecInit((0 until Width).map(i => enqPtr + i.U)) 233 val enqPtrVec = VecInit((0 until Width).map(i => enqPtrVecInit(accumEnqNum(i)))) 234 val enqNum = PopCount(reqs.map(_.valid)) 235 canEnqueue := counter +& enqNum <= Size.U 236 237 // the req may recv false ready, but actually received. Filter and TLB will handle it. 238 val enqNum_fake = PopCount(io.tlb.req.map(_.valid)) 239 val canEnqueue_fake = counter +& enqNum_fake <= Size.U 240 io.tlb.req.map(_.ready := canEnqueue_fake) // NOTE: just drop un-fire reqs 241 242 // tlb req flushed by ptw resp: last ptw resp && current ptw resp 243 // the flushed tlb req will fakely enq, with a false valid 244 val tlb_req_flushed = reqs.map(a => io.ptw.resp.valid && io.ptw.resp.bits.hit(a.bits.vpn, 0.U, true, true)) 245 246 io.tlb.resp.valid := ptwResp_valid 247 io.tlb.resp.bits.data.entry := ptwResp.entry 248 io.tlb.resp.bits.data.addr_low := ptwResp.addr_low 249 io.tlb.resp.bits.data.ppn_low := ptwResp.ppn_low 250 io.tlb.resp.bits.data.valididx := ptwResp.valididx 251 io.tlb.resp.bits.data.pteidx := ptwResp.pteidx 252 io.tlb.resp.bits.data.pf := ptwResp.pf 253 io.tlb.resp.bits.data.af := ptwResp.af 254 io.tlb.resp.bits.data.memidx := memidx(OHToUInt(ptwResp_OldMatchVec)) 255 io.tlb.resp.bits.vector := resp_vector 256 257 val issue_valid = v(issPtr) && !isEmptyIss && !inflight_full 258 val issue_filtered = ptwResp_valid && ptwResp.hit(io.ptw.req(0).bits.vpn, io.csr.satp.asid, allType=true, ignoreAsid=true) 259 val issue_fire_fake = issue_valid && (io.ptw.req(0).ready || (issue_filtered && false.B /*timing-opt*/)) 260 io.ptw.req(0).valid := issue_valid && !issue_filtered 261 io.ptw.req(0).bits.vpn := vpn(issPtr) 262 io.ptw.resp.ready := true.B 263 264 reqs.zipWithIndex.map{ 265 case (req, i) => 266 when (req.valid && canEnqueue) { 267 v(enqPtrVec(i)) := !tlb_req_flushed(i) 268 vpn(enqPtrVec(i)) := req.bits.vpn 269 memidx(enqPtrVec(i)) := req.bits.memidx 270 ports(enqPtrVec(i)) := req_ports(i).asBools 271 } 272 } 273 for (i <- ports.indices) { 274 when (v(i)) { 275 ports(i) := ports(i).zip(update_ports(i)).map(a => a._1 || a._2) 276 } 277 } 278 279 val do_enq = canEnqueue && Cat(reqs.map(_.valid)).orR 280 val do_deq = (!v(deqPtr) && !isEmptyDeq) 281 val do_iss = issue_fire_fake || (!v(issPtr) && !isEmptyIss) 282 when (do_enq) { 283 enqPtr := enqPtr + enqNum 284 } 285 when (do_deq) { 286 deqPtr := deqPtr + 1.U 287 } 288 when (do_iss) { 289 issPtr := issPtr + 1.U 290 } 291 when (issue_fire_fake && issue_filtered) { // issued but is filtered 292 v(issPtr) := false.B 293 } 294 when (do_enq =/= do_deq) { 295 mayFullDeq := do_enq 296 } 297 when (do_enq =/= do_iss) { 298 mayFullIss := do_enq 299 } 300 301 when (io.ptw.resp.fire()) { 302 v.zip(ptwResp_OldMatchVec).map{ case (vi, mi) => when (mi) { vi := false.B }} 303 } 304 305 counter := counter - do_deq + Mux(do_enq, enqNum, 0.U) 306 assert(counter <= Size.U, "counter should be no more than Size") 307 assert(inflight_counter <= Size.U, "inflight should be no more than Size") 308 when (counter === 0.U) { 309 assert(!io.ptw.req(0).fire(), "when counter is 0, should not req") 310 assert(isEmptyDeq && isEmptyIss, "when counter is 0, should be empty") 311 } 312 when (counter === Size.U) { 313 assert(mayFullDeq, "when counter is Size, should be full") 314 } 315 316 when (flush) { 317 v.map(_ := false.B) 318 deqPtr := 0.U 319 enqPtr := 0.U 320 issPtr := 0.U 321 ptwResp_valid := false.B 322 mayFullDeq := false.B 323 mayFullIss := false.B 324 counter := 0.U 325 inflight_counter := 0.U 326 } 327 328 val sourceVaddr = WireInit(0.U.asTypeOf(new Valid(UInt(VAddrBits.W)))) 329 330 ExcitingUtils.addSink(sourceVaddr, s"rob_head_vaddr_${coreParams.HartId}", ExcitingUtils.Perf) 331 332 io.rob_head_miss_in_tlb := VecInit(v.zip(vpn).map{case (vi, vpni) => { 333 vi && sourceVaddr.valid && vpni === get_pn(sourceVaddr.bits) 334 }}).asUInt.orR 335 336 // perf 337 XSPerfAccumulate("tlb_req_count", PopCount(Cat(io.tlb.req.map(_.valid)))) 338 XSPerfAccumulate("tlb_req_count_filtered", Mux(do_enq, accumEnqNum(Width - 1), 0.U)) 339 XSPerfAccumulate("ptw_req_count", io.ptw.req(0).fire()) 340 XSPerfAccumulate("ptw_req_cycle", inflight_counter) 341 XSPerfAccumulate("tlb_resp_count", io.tlb.resp.fire()) 342 XSPerfAccumulate("ptw_resp_count", io.ptw.resp.fire()) 343 XSPerfAccumulate("inflight_cycle", !isEmptyDeq) 344 for (i <- 0 until Size + 1) { 345 XSPerfAccumulate(s"counter${i}", counter === i.U) 346 } 347 348 for (i <- 0 until Size) { 349 TimeOutAssert(v(i), timeOutThreshold, s"Filter ${i} doesn't recv resp in time") 350 } 351} 352 353object PTWRepeater { 354 def apply(fenceDelay: Int, 355 tlb: TlbPtwIO, 356 sfence: SfenceBundle, 357 csr: TlbCsrBundle 358 )(implicit p: Parameters) = { 359 val width = tlb.req.size 360 val repeater = Module(new PTWRepeater(width, fenceDelay)) 361 repeater.io.apply(tlb, sfence, csr) 362 repeater 363 } 364 365 def apply(fenceDelay: Int, 366 tlb: TlbPtwIO, 367 ptw: TlbPtwIO, 368 sfence: SfenceBundle, 369 csr: TlbCsrBundle 370 )(implicit p: Parameters) = { 371 val width = tlb.req.size 372 val repeater = Module(new PTWRepeater(width, fenceDelay)) 373 repeater.io.apply(tlb, ptw, sfence, csr) 374 repeater 375 } 376} 377 378object PTWRepeaterNB { 379 def apply(passReady: Boolean, fenceDelay: Int, 380 tlb: TlbPtwIO, 381 sfence: SfenceBundle, 382 csr: TlbCsrBundle 383 )(implicit p: Parameters) = { 384 val width = tlb.req.size 385 val repeater = Module(new PTWRepeaterNB(width, passReady,fenceDelay)) 386 repeater.io.apply(tlb, sfence, csr) 387 repeater 388 } 389 390 def apply(passReady: Boolean, fenceDelay: Int, 391 tlb: TlbPtwIO, 392 ptw: TlbPtwIO, 393 sfence: SfenceBundle, 394 csr: TlbCsrBundle 395 )(implicit p: Parameters) = { 396 val width = tlb.req.size 397 val repeater = Module(new PTWRepeaterNB(width, passReady, fenceDelay)) 398 repeater.io.apply(tlb, ptw, sfence, csr) 399 repeater 400 } 401} 402 403object PTWFilter { 404 def apply(fenceDelay: Int, 405 tlb: VectorTlbPtwIO, 406 ptw: TlbPtwIO, 407 sfence: SfenceBundle, 408 csr: TlbCsrBundle, 409 size: Int 410 )(implicit p: Parameters) = { 411 val width = tlb.req.size 412 val filter = Module(new PTWFilter(width, size, fenceDelay)) 413 filter.io.apply(tlb, ptw, sfence, csr) 414 filter 415 } 416 417 def apply(fenceDelay: Int, 418 tlb: VectorTlbPtwIO, 419 sfence: SfenceBundle, 420 csr: TlbCsrBundle, 421 size: Int 422 )(implicit p: Parameters) = { 423 val width = tlb.req.size 424 val filter = Module(new PTWFilter(width, size, fenceDelay)) 425 filter.io.apply(tlb, sfence, csr) 426 filter 427 } 428 429} 430