1/*************************************************************************************** 2* Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences 3* Copyright (c) 2020-2021 Peng Cheng Laboratory 4* 5* XiangShan is licensed under Mulan PSL v2. 6* You can use this software according to the terms and conditions of the Mulan PSL v2. 7* You may obtain a copy of Mulan PSL v2 at: 8* http://license.coscl.org.cn/MulanPSL2 9* 10* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, 11* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, 12* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. 13* 14* See the Mulan PSL v2 for more details. 15***************************************************************************************/ 16 17package xiangshan.cache.mmu 18 19import org.chipsalliance.cde.config.Parameters 20import chisel3._ 21import chisel3.util._ 22import xiangshan._ 23import xiangshan.cache.{HasDCacheParameters, MemoryOpConstants} 24import utils._ 25import utility._ 26import freechips.rocketchip.diplomacy.{LazyModule, LazyModuleImp} 27import freechips.rocketchip.tilelink._ 28 29class PTWReapterIO(Width: Int)(implicit p: Parameters) extends MMUIOBaseBundle { 30 val tlb = Flipped(new TlbPtwIO(Width)) 31 val ptw = new TlbPtwIO 32 33 def apply(tlb: TlbPtwIO, ptw: TlbPtwIO, sfence: SfenceBundle, csr: TlbCsrBundle): Unit = { 34 this.tlb <> tlb 35 this.ptw <> ptw 36 this.sfence <> sfence 37 this.csr <> csr 38 } 39 40 def apply(tlb: TlbPtwIO, sfence: SfenceBundle, csr: TlbCsrBundle): Unit = { 41 this.tlb <> tlb 42 this.sfence <> sfence 43 this.csr <> csr 44 } 45 46} 47 48class PTWRepeater(Width: Int = 1, FenceDelay: Int)(implicit p: Parameters) extends XSModule with HasPtwConst { 49 val io = IO(new PTWReapterIO(Width)) 50 51 val req_in = if (Width == 1) { 52 io.tlb.req(0) 53 } else { 54 val arb = Module(new RRArbiter(io.tlb.req(0).bits.cloneType, Width)) 55 arb.io.in <> io.tlb.req 56 arb.io.out 57 } 58 val (tlb, ptw, flush) = (io.tlb, io.ptw, DelayN(io.sfence.valid || io.csr.satp.changed, FenceDelay)) 59 val req = RegEnable(req_in.bits, req_in.fire) 60 val resp = RegEnable(ptw.resp.bits, ptw.resp.fire) 61 val haveOne = BoolStopWatch(req_in.fire, tlb.resp.fire || flush) 62 val sent = BoolStopWatch(ptw.req(0).fire, req_in.fire || flush) 63 val recv = BoolStopWatch(ptw.resp.fire && haveOne, req_in.fire || flush) 64 65 req_in.ready := !haveOne 66 ptw.req(0).valid := haveOne && !sent 67 ptw.req(0).bits := req 68 69 tlb.resp.bits := resp 70 tlb.resp.valid := haveOne && recv 71 ptw.resp.ready := !recv 72 73 XSPerfAccumulate("req_count", ptw.req(0).fire) 74 XSPerfAccumulate("tlb_req_cycle", BoolStopWatch(req_in.fire, tlb.resp.fire || flush)) 75 XSPerfAccumulate("ptw_req_cycle", BoolStopWatch(ptw.req(0).fire, ptw.resp.fire || flush)) 76 77 XSDebug(haveOne, p"haveOne:${haveOne} sent:${sent} recv:${recv} sfence:${flush} req:${req} resp:${resp}") 78 XSDebug(req_in.valid || io.tlb.resp.valid, p"tlb: ${tlb}\n") 79 XSDebug(io.ptw.req(0).valid || io.ptw.resp.valid, p"ptw: ${ptw}\n") 80 assert(!RegNext(recv && io.ptw.resp.valid, init = false.B), "re-receive ptw.resp") 81 XSError(io.ptw.req(0).valid && io.ptw.resp.valid && !flush, "ptw repeater recv resp when sending") 82 XSError(io.ptw.resp.valid && (req.vpn =/= io.ptw.resp.bits.entry.tag), "ptw repeater recv resp with wrong tag") 83 XSError(io.ptw.resp.valid && !io.ptw.resp.ready, "ptw repeater's ptw resp back, but not ready") 84 TimeOutAssert(sent && !recv, timeOutThreshold, "Repeater doesn't recv resp in time") 85} 86 87/* dtlb 88 * 89 */ 90 91class PTWRepeaterNB(Width: Int = 1, passReady: Boolean = false, FenceDelay: Int)(implicit p: Parameters) extends XSModule with HasPtwConst { 92 val io = IO(new PTWReapterIO(Width)) 93 94 val req_in = if (Width == 1) { 95 io.tlb.req(0) 96 } else { 97 val arb = Module(new RRArbiter(io.tlb.req(0).bits.cloneType, Width)) 98 arb.io.in <> io.tlb.req 99 arb.io.out 100 } 101 val (tlb, ptw, flush) = (io.tlb, io.ptw, DelayN(io.sfence.valid || io.csr.satp.changed, FenceDelay)) 102 /* sent: tlb -> repeater -> ptw 103 * recv: ptw -> repeater -> tlb 104 * different from PTWRepeater 105 */ 106 107 // tlb -> repeater -> ptw 108 val req = RegEnable(req_in.bits, req_in.fire) 109 val sent = BoolStopWatch(req_in.fire, ptw.req(0).fire || flush) 110 req_in.ready := !sent || { if (passReady) ptw.req(0).ready else false.B } 111 ptw.req(0).valid := sent 112 ptw.req(0).bits := req 113 114 // ptw -> repeater -> tlb 115 val resp = RegEnable(ptw.resp.bits, ptw.resp.fire) 116 val recv = BoolStopWatch(ptw.resp.fire, tlb.resp.fire || flush) 117 ptw.resp.ready := !recv || { if (passReady) tlb.resp.ready else false.B } 118 tlb.resp.valid := recv 119 tlb.resp.bits := resp 120 121 XSPerfAccumulate("req", req_in.fire) 122 XSPerfAccumulate("resp", tlb.resp.fire) 123 if (!passReady) { 124 XSPerfAccumulate("req_blank", req_in.valid && sent && ptw.req(0).ready) 125 XSPerfAccumulate("resp_blank", ptw.resp.valid && recv && tlb.resp.ready) 126 XSPerfAccumulate("req_blank_ignore_ready", req_in.valid && sent) 127 XSPerfAccumulate("resp_blank_ignore_ready", ptw.resp.valid && recv) 128 } 129 XSDebug(req_in.valid || io.tlb.resp.valid, p"tlb: ${tlb}\n") 130 XSDebug(io.ptw.req(0).valid || io.ptw.resp.valid, p"ptw: ${ptw}\n") 131} 132 133class PTWFilterIO(Width: Int, hasHint: Boolean = false)(implicit p: Parameters) extends MMUIOBaseBundle { 134 val tlb = Flipped(new VectorTlbPtwIO(Width)) 135 val ptw = new TlbPtwIO() 136 val hint = if (hasHint) Some(new TlbHintIO) else None 137 val rob_head_miss_in_tlb = Output(Bool()) 138 val debugTopDown = new Bundle { 139 val robHeadVaddr = Flipped(Valid(UInt(VAddrBits.W))) 140 } 141 142 def apply(tlb: VectorTlbPtwIO, ptw: TlbPtwIO, sfence: SfenceBundle, csr: TlbCsrBundle): Unit = { 143 this.tlb <> tlb 144 this.ptw <> ptw 145 this.sfence <> sfence 146 this.csr <> csr 147 } 148 149 def apply(tlb: VectorTlbPtwIO, sfence: SfenceBundle, csr: TlbCsrBundle): Unit = { 150 this.tlb <> tlb 151 this.sfence <> sfence 152 this.csr <> csr 153 } 154 155} 156 157class PTWFilterEntryIO(Width: Int, hasHint: Boolean = false)(implicit p: Parameters) extends PTWFilterIO(Width, hasHint){ 158 val flush = Input(Bool()) 159 val refill = Output(Bool()) 160 val memidx = Output(new MemBlockidxBundle) 161} 162 163class PTWFilterEntry(Width: Int, Size: Int, hasHint: Boolean = false)(implicit p: Parameters) extends XSModule with HasPtwConst { 164 165 val io = IO(new PTWFilterEntryIO(Width, hasHint)) 166 require(isPow2(Size), s"Filter Size ($Size) must be a power of 2") 167 168 def firstValidIndex(v: Seq[Bool], valid: Bool): UInt = { 169 val index = WireInit(0.U(log2Up(Size).W)) 170 for (i <- 0 until v.size) { 171 when (v(i) === valid) { 172 index := i.U 173 } 174 } 175 index 176 } 177 178 val v = RegInit(VecInit(Seq.fill(Size)(false.B))) 179 val sent = RegInit(VecInit(Seq.fill(Size)(false.B))) 180 val vpn = Reg(Vec(Size, UInt(vpnLen.W))) 181 val memidx = Reg(Vec(Size, new MemBlockidxBundle)) 182 183 val enqvalid = WireInit(VecInit(Seq.fill(Width)(false.B))) 184 val canenq = WireInit(VecInit(Seq.fill(Width)(false.B))) 185 val enqidx = WireInit(VecInit(Seq.fill(Width)(0.U(log2Up(Size).W)))) 186 187 //val selectCount = RegInit(0.U(log2Up(Width).W)) 188 189 val entryIsMatchVec = WireInit(VecInit(Seq.fill(Width)(false.B))) 190 val entryMatchIndexVec = WireInit(VecInit(Seq.fill(Width)(0.U(log2Up(Size).W)))) 191 val ptwResp_EntryMatchVec = vpn.zip(v).map{ case (pi, vi) => vi && io.ptw.resp.bits.hit(pi, io.csr.satp.asid, true, true)} 192 val ptwResp_EntryMatchFirst = firstValidIndex(ptwResp_EntryMatchVec, true.B) 193 val ptwResp_ReqMatchVec = io.tlb.req.map(a => io.ptw.resp.valid && io.ptw.resp.bits.hit(a.bits.vpn, 0.U, allType = true, true)) 194 195 io.refill := Cat(ptwResp_EntryMatchVec).orR && io.ptw.resp.fire 196 io.ptw.resp.ready := true.B 197 // DontCare 198 io.tlb.req.map(_.ready := true.B) 199 io.tlb.resp.valid := false.B 200 io.tlb.resp.bits.data := 0.U.asTypeOf(new PtwSectorRespwithMemIdx) 201 io.tlb.resp.bits.vector := 0.U.asTypeOf(Vec(Width, Bool())) 202 io.memidx := 0.U.asTypeOf(new MemBlockidxBundle) 203 204 // ugly code, should be optimized later 205 if (Enable3Load3Store) { 206 require(Width <= 4, s"DTLB Filter Width ($Width) must equal or less than 4") 207 if (Width == 1) { 208 require(Size == 8, s"prefetch filter Size ($Size) should be 8") 209 canenq(0) := !(Cat(v).andR) 210 enqidx(0) := firstValidIndex(v, false.B) 211 } else if (Width == 3) { 212 require(Size == 8, s"store filter Size ($Size) should be 8") 213 canenq(0) := !(Cat(v.take(3)).andR) 214 enqidx(0) := firstValidIndex(v.take(3), false.B) 215 canenq(1) := !(Cat(v.drop(3).take(3)).andR) 216 enqidx(1) := firstValidIndex(v.drop(3).take(3), false.B) + 3.U 217 canenq(2) := !(Cat(v.drop(6).take(2)).andR) 218 enqidx(2) := firstValidIndex(v.drop(6).take(2), false.B) + 6.U 219 } else if (Width == 4) { 220 require(Size == 16, s"load filter Size ($Size) should be 16") 221 canenq(0) := !(Cat(v.take(4)).andR) 222 enqidx(0) := firstValidIndex(v.take(4), false.B) 223 canenq(1) := !(Cat(v.drop(4).take(4)).andR) 224 enqidx(1) := firstValidIndex(v.drop(4).take(4), false.B) + 4.U 225 canenq(2) := !(Cat(v.drop(8).take(4)).andR) 226 enqidx(2) := firstValidIndex(v.drop(8).take(4), false.B) + 8.U 227 canenq(3) := !(Cat(v.drop(12).take(4)).andR) 228 enqidx(3) := firstValidIndex(v.drop(12).take(4), false.B) + 12.U 229 } 230 } else { 231 require(Width <= 3, s"DTLB Filter Width ($Width) must equal or less than 3") 232 if (Width == 1) { 233 require(Size == 8, s"prefetch filter Size ($Size) should be 8") 234 canenq(0) := !(Cat(v).andR) 235 enqidx(0) := firstValidIndex(v, false.B) 236 } else if (Width == 2) { 237 require(Size == 8, s"store filter Size ($Size) should be 8") 238 canenq(0) := !(Cat(v.take(Size/2)).andR) 239 enqidx(0) := firstValidIndex(v.take(Size/2), false.B) 240 canenq(1) := !(Cat(v.drop(Size/2)).andR) 241 enqidx(1) := firstValidIndex(v.drop(Size/2), false.B) + (Size/2).U 242 } else if (Width == 3) { 243 require(Size == 16, s"load filter Size ($Size) should be 16") 244 canenq(0) := !(Cat(v.take(8)).andR) 245 enqidx(0) := firstValidIndex(v.take(8), false.B) 246 canenq(1) := !(Cat(v.drop(8).take(4)).andR) 247 enqidx(1) := firstValidIndex(v.drop(8).take(4), false.B) + 8.U 248 // four entries for prefetch 249 canenq(2) := !(Cat(v.drop(12)).andR) 250 enqidx(2) := firstValidIndex(v.drop(12), false.B) + 12.U 251 } 252 } 253 254 255 for (i <- 0 until Width) { 256 enqvalid(i) := io.tlb.req(i).valid && !ptwResp_ReqMatchVec(i) && !entryIsMatchVec(i) && canenq(i) 257 when (!enqvalid(i)) { 258 enqidx(i) := entryMatchIndexVec(i) 259 } 260 261 val entryIsMatch = vpn.zip(v).map{ case (pi, vi) => vi && pi === io.tlb.req(i).bits.vpn} 262 entryIsMatchVec(i) := Cat(entryIsMatch).orR 263 entryMatchIndexVec(i) := firstValidIndex(entryIsMatch, true.B) 264 265 if (i > 0) { 266 for (j <- 0 until i) { 267 val newIsMatch = io.tlb.req(i).bits.vpn === io.tlb.req(j).bits.vpn 268 when (newIsMatch && io.tlb.req(j).valid) { 269 enqidx(i) := enqidx(j) 270 canenq(i) := canenq(j) 271 enqvalid(i) := false.B 272 } 273 } 274 } 275 276 when (enqvalid(i)) { 277 v(enqidx(i)) := true.B 278 sent(enqidx(i)) := false.B 279 vpn(enqidx(i)) := io.tlb.req(i).bits.vpn 280 memidx(enqidx(i)) := io.tlb.req(i).bits.memidx 281 } 282 } 283 284 val issuevec = v.zip(sent).map{ case (v, s) => v && !s} 285 val issueindex = firstValidIndex(issuevec, true.B) 286 val canissue = Cat(issuevec).orR 287 for (i <- 0 until Size) { 288 io.ptw.req(0).valid := canissue 289 io.ptw.req(0).bits.vpn := vpn(issueindex) 290 } 291 when (io.ptw.req(0).fire) { 292 sent(issueindex) := true.B 293 } 294 295 when (io.ptw.resp.fire) { 296 v.zip(ptwResp_EntryMatchVec).map{ case (vi, mi) => when (mi) { vi := false.B }} 297 io.memidx := memidx(ptwResp_EntryMatchFirst) 298 } 299 300 when (io.flush) { 301 v.map(_ := false.B) 302 } 303 304 if (hasHint) { 305 val hintIO = io.hint.getOrElse(new TlbHintIO) 306 for (i <- 0 until exuParameters.LduCnt) { 307 hintIO.req(i).id := enqidx(i) 308 hintIO.req(i).full := !canenq(i) || ptwResp_ReqMatchVec(i) 309 } 310 hintIO.resp.valid := io.refill 311 hintIO.resp.bits.id := ptwResp_EntryMatchFirst 312 hintIO.resp.bits.replay_all := PopCount(ptwResp_EntryMatchVec) > 1.U 313 } 314 315 io.rob_head_miss_in_tlb := VecInit(v.zip(vpn).map{case (vi, vpni) => { 316 vi && io.debugTopDown.robHeadVaddr.valid && vpni === get_pn(io.debugTopDown.robHeadVaddr.bits) 317 }}).asUInt.orR 318 319 320 // Perf Counter 321 val counter = PopCount(v) 322 val inflight_counter = RegInit(0.U(log2Up(Size).W)) 323 val inflight_full = inflight_counter === Size.U 324 when (io.ptw.req(0).fire =/= io.ptw.resp.fire) { 325 inflight_counter := Mux(io.ptw.req(0).fire, inflight_counter + 1.U, inflight_counter - 1.U) 326 } 327 328 assert(inflight_counter <= Size.U, "inflight should be no more than Size") 329 when (counter === 0.U) { 330 assert(!io.ptw.req(0).fire, "when counter is 0, should not req") 331 } 332 333 when (io.flush) { 334 inflight_counter := 0.U 335 } 336 337 XSPerfAccumulate("tlb_req_count", PopCount(Cat(io.tlb.req.map(_.valid)))) 338 XSPerfAccumulate("tlb_req_count_filtered", PopCount(enqvalid)) 339 XSPerfAccumulate("ptw_req_count", io.ptw.req(0).fire) 340 XSPerfAccumulate("ptw_req_cycle", inflight_counter) 341 XSPerfAccumulate("tlb_resp_count", io.tlb.resp.fire) 342 XSPerfAccumulate("ptw_resp_count", io.ptw.resp.fire) 343 XSPerfAccumulate("inflight_cycle", Cat(sent).orR) 344 345 for (i <- 0 until Size + 1) { 346 XSPerfAccumulate(s"counter${i}", counter === i.U) 347 } 348 349 for (i <- 0 until Size) { 350 TimeOutAssert(v(i), timeOutThreshold, s"Filter ${i} doesn't recv resp in time") 351 } 352 353} 354 355class PTWNewFilter(Width: Int, Size: Int, FenceDelay: Int)(implicit p: Parameters) extends XSModule with HasPtwConst { 356 require(Size >= Width) 357 358 val io = IO(new PTWFilterIO(Width, hasHint = true)) 359 360 val load_filter = VecInit(Seq.fill(1) { 361 val load_entry = Module(new PTWFilterEntry(Width = exuParameters.LduCnt + 1, Size = loadfiltersize, hasHint = true)) 362 load_entry.io 363 }) 364 365 val store_filter = VecInit(Seq.fill(1) { 366 val store_entry = Module(new PTWFilterEntry(Width = exuParameters.StuCnt, Size = storefiltersize)) 367 store_entry.io 368 }) 369 370 val prefetch_filter = VecInit(Seq.fill(1) { 371 val prefetch_entry = Module(new PTWFilterEntry(Width = 1, Size = prefetchfiltersize)) 372 prefetch_entry.io 373 }) 374 375 val filter = load_filter ++ store_filter ++ prefetch_filter 376 377 load_filter.map(_.tlb.req := io.tlb.req.take(exuParameters.LduCnt + 1)) 378 store_filter.map(_.tlb.req := io.tlb.req.drop(exuParameters.LduCnt + 1).take(exuParameters.StuCnt)) 379 prefetch_filter.map(_.tlb.req := io.tlb.req.drop(exuParameters.LduCnt + 1 + exuParameters.StuCnt)) 380 381 val flush = DelayN(io.sfence.valid || io.csr.satp.changed, FenceDelay) 382 val ptwResp = RegEnable(io.ptw.resp.bits, io.ptw.resp.fire) 383 val ptwResp_valid = Cat(filter.map(_.refill)).orR 384 filter.map(_.tlb.resp.ready := true.B) 385 filter.map(_.ptw.resp.valid := RegNext(io.ptw.resp.fire, init = false.B)) 386 filter.map(_.ptw.resp.bits := ptwResp) 387 filter.map(_.flush := flush) 388 filter.map(_.sfence := io.sfence) 389 filter.map(_.csr := io.csr) 390 filter.map(_.debugTopDown.robHeadVaddr := io.debugTopDown.robHeadVaddr) 391 392 io.tlb.req.map(_.ready := true.B) 393 io.tlb.resp.valid := ptwResp_valid 394 io.tlb.resp.bits.data.entry := ptwResp.entry 395 io.tlb.resp.bits.data.addr_low := ptwResp.addr_low 396 io.tlb.resp.bits.data.ppn_low := ptwResp.ppn_low 397 io.tlb.resp.bits.data.valididx := ptwResp.valididx 398 io.tlb.resp.bits.data.pteidx := ptwResp.pteidx 399 io.tlb.resp.bits.data.pf := ptwResp.pf 400 io.tlb.resp.bits.data.af := ptwResp.af 401 io.tlb.resp.bits.data.memidx := 0.U.asTypeOf(new MemBlockidxBundle) 402 // vector used to represent different requestors of DTLB 403 // (e.g. the store DTLB has StuCnt requestors) 404 // However, it is only necessary to distinguish between different DTLB now 405 for (i <- 0 until Width) { 406 io.tlb.resp.bits.vector(i) := false.B 407 } 408 io.tlb.resp.bits.vector(0) := load_filter(0).refill 409 io.tlb.resp.bits.vector(exuParameters.LduCnt + 1) := store_filter(0).refill 410 io.tlb.resp.bits.vector(exuParameters.LduCnt + 1 + exuParameters.StuCnt) := prefetch_filter(0).refill 411 412 val hintIO = io.hint.getOrElse(new TlbHintIO) 413 val load_hintIO = load_filter(0).hint.getOrElse(new TlbHintIO) 414 for (i <- 0 until exuParameters.LduCnt) { 415 hintIO.req(i) := RegNext(load_hintIO.req(i)) 416 } 417 hintIO.resp := RegNext(load_hintIO.resp) 418 419 when (load_filter(0).refill) { 420 io.tlb.resp.bits.vector(0) := true.B 421 io.tlb.resp.bits.data.memidx := load_filter(0).memidx 422 } 423 when (store_filter(0).refill) { 424 io.tlb.resp.bits.vector(exuParameters.LduCnt + 1) := true.B 425 io.tlb.resp.bits.data.memidx := store_filter(0).memidx 426 } 427 when (prefetch_filter(0).refill) { 428 io.tlb.resp.bits.vector(exuParameters.LduCnt + 1 + exuParameters.StuCnt) := true.B 429 io.tlb.resp.bits.data.memidx := 0.U.asTypeOf(new MemBlockidxBundle) 430 } 431 432 val ptw_arb = Module(new RRArbiterInit(new PtwReq, 3)) 433 for (i <- 0 until 3) { 434 ptw_arb.io.in(i).valid := filter(i).ptw.req(0).valid 435 ptw_arb.io.in(i).bits.vpn := filter(i).ptw.req(0).bits.vpn 436 filter(i).ptw.req(0).ready := ptw_arb.io.in(i).ready 437 } 438 ptw_arb.io.out.ready := io.ptw.req(0).ready 439 io.ptw.req(0).valid := ptw_arb.io.out.valid 440 io.ptw.req(0).bits.vpn := ptw_arb.io.out.bits.vpn 441 io.ptw.resp.ready := true.B 442 443 io.rob_head_miss_in_tlb := Cat(filter.map(_.rob_head_miss_in_tlb)).orR 444} 445 446class PTWFilter(Width: Int, Size: Int, FenceDelay: Int)(implicit p: Parameters) extends XSModule with HasPtwConst { 447 require(Size >= Width) 448 449 val io = IO(new PTWFilterIO(Width)) 450 451 val v = RegInit(VecInit(Seq.fill(Size)(false.B))) 452 val ports = Reg(Vec(Size, Vec(Width, Bool()))) // record which port(s) the entry come from, may not able to cover all the ports 453 val vpn = Reg(Vec(Size, UInt(vpnLen.W))) 454 val memidx = Reg(Vec(Size, new MemBlockidxBundle)) 455 val enqPtr = RegInit(0.U(log2Up(Size).W)) // Enq 456 val issPtr = RegInit(0.U(log2Up(Size).W)) // Iss to Ptw 457 val deqPtr = RegInit(0.U(log2Up(Size).W)) // Deq 458 val mayFullDeq = RegInit(false.B) 459 val mayFullIss = RegInit(false.B) 460 val counter = RegInit(0.U(log2Up(Size+1).W)) 461 462 val flush = DelayN(io.sfence.valid || io.csr.satp.changed, FenceDelay) 463 val tlb_req = WireInit(io.tlb.req) // NOTE: tlb_req is not io.tlb.req, see below codes, just use cloneType 464 tlb_req.suggestName("tlb_req") 465 466 val inflight_counter = RegInit(0.U(log2Up(Size + 1).W)) 467 val inflight_full = inflight_counter === Size.U 468 when (io.ptw.req(0).fire =/= io.ptw.resp.fire) { 469 inflight_counter := Mux(io.ptw.req(0).fire, inflight_counter + 1.U, inflight_counter - 1.U) 470 } 471 472 val canEnqueue = Wire(Bool()) // NOTE: actually enqueue 473 val ptwResp = RegEnable(io.ptw.resp.bits, io.ptw.resp.fire) 474 val ptwResp_OldMatchVec = vpn.zip(v).map{ case (pi, vi) => 475 vi && io.ptw.resp.bits.hit(pi, io.csr.satp.asid, true, true)} 476 val ptwResp_valid = RegNext(io.ptw.resp.fire && Cat(ptwResp_OldMatchVec).orR, init = false.B) 477 // May send repeated requests to L2 tlb with same vpn(26, 3) when sector tlb 478 val oldMatchVec_early = io.tlb.req.map(a => vpn.zip(v).map{ case (pi, vi) => vi && pi === a.bits.vpn}) 479 val lastReqMatchVec_early = io.tlb.req.map(a => tlb_req.map{ b => b.valid && b.bits.vpn === a.bits.vpn && canEnqueue}) 480 val newMatchVec_early = io.tlb.req.map(a => io.tlb.req.map(b => a.bits.vpn === b.bits.vpn)) 481 482 (0 until Width) foreach { i => 483 tlb_req(i).valid := RegNext(io.tlb.req(i).valid && 484 !(ptwResp_valid && ptwResp.hit(io.tlb.req(i).bits.vpn, 0.U, true, true)) && 485 !Cat(lastReqMatchVec_early(i)).orR, 486 init = false.B) 487 tlb_req(i).bits := RegEnable(io.tlb.req(i).bits, io.tlb.req(i).valid) 488 } 489 490 val oldMatchVec = oldMatchVec_early.map(a => RegNext(Cat(a).orR)) 491 val newMatchVec = (0 until Width).map(i => (0 until Width).map(j => 492 RegNext(newMatchVec_early(i)(j)) && tlb_req(j).valid 493 )) 494 val ptwResp_newMatchVec = tlb_req.map(a => 495 ptwResp_valid && ptwResp.hit(a.bits.vpn, 0.U, allType = true, true)) 496 497 val oldMatchVec2 = (0 until Width).map(i => oldMatchVec_early(i).map(RegNext(_)).map(_ & tlb_req(i).valid)) 498 val update_ports = v.indices.map(i => oldMatchVec2.map(j => j(i))) 499 val ports_init = (0 until Width).map(i => (1 << i).U(Width.W)) 500 val filter_ports = (0 until Width).map(i => ParallelMux(newMatchVec(i).zip(ports_init).drop(i))) 501 val resp_vector = RegEnable(ParallelMux(ptwResp_OldMatchVec zip ports), io.ptw.resp.fire) 502 503 def canMerge(index: Int) : Bool = { 504 ptwResp_newMatchVec(index) || oldMatchVec(index) || 505 Cat(newMatchVec(index).take(index)).orR 506 } 507 508 def filter_req() = { 509 val reqs = tlb_req.indices.map{ i => 510 val req = Wire(ValidIO(new PtwReqwithMemIdx())) 511 val merge = canMerge(i) 512 req.bits := tlb_req(i).bits 513 req.valid := !merge && tlb_req(i).valid 514 req 515 } 516 reqs 517 } 518 519 val reqs = filter_req() 520 val req_ports = filter_ports 521 val isFull = enqPtr === deqPtr && mayFullDeq 522 val isEmptyDeq = enqPtr === deqPtr && !mayFullDeq 523 val isEmptyIss = enqPtr === issPtr && !mayFullIss 524 val accumEnqNum = (0 until Width).map(i => PopCount(reqs.take(i).map(_.valid))) 525 val enqPtrVecInit = VecInit((0 until Width).map(i => enqPtr + i.U)) 526 val enqPtrVec = VecInit((0 until Width).map(i => enqPtrVecInit(accumEnqNum(i)))) 527 val enqNum = PopCount(reqs.map(_.valid)) 528 canEnqueue := counter +& enqNum <= Size.U 529 530 // the req may recv false ready, but actually received. Filter and TLB will handle it. 531 val enqNum_fake = PopCount(io.tlb.req.map(_.valid)) 532 val canEnqueue_fake = counter +& enqNum_fake <= Size.U 533 io.tlb.req.map(_.ready := canEnqueue_fake) // NOTE: just drop un-fire reqs 534 535 // tlb req flushed by ptw resp: last ptw resp && current ptw resp 536 // the flushed tlb req will fakely enq, with a false valid 537 val tlb_req_flushed = reqs.map(a => io.ptw.resp.valid && io.ptw.resp.bits.hit(a.bits.vpn, 0.U, true, true)) 538 539 io.tlb.resp.valid := ptwResp_valid 540 io.tlb.resp.bits.data.entry := ptwResp.entry 541 io.tlb.resp.bits.data.addr_low := ptwResp.addr_low 542 io.tlb.resp.bits.data.ppn_low := ptwResp.ppn_low 543 io.tlb.resp.bits.data.valididx := ptwResp.valididx 544 io.tlb.resp.bits.data.pteidx := ptwResp.pteidx 545 io.tlb.resp.bits.data.pf := ptwResp.pf 546 io.tlb.resp.bits.data.af := ptwResp.af 547 io.tlb.resp.bits.data.memidx := memidx(OHToUInt(ptwResp_OldMatchVec)) 548 io.tlb.resp.bits.vector := resp_vector 549 550 val issue_valid = v(issPtr) && !isEmptyIss && !inflight_full 551 val issue_filtered = ptwResp_valid && ptwResp.hit(io.ptw.req(0).bits.vpn, io.csr.satp.asid, allType=true, ignoreAsid=true) 552 val issue_fire_fake = issue_valid && (io.ptw.req(0).ready || (issue_filtered && false.B /*timing-opt*/)) 553 io.ptw.req(0).valid := issue_valid && !issue_filtered 554 io.ptw.req(0).bits.vpn := vpn(issPtr) 555 io.ptw.resp.ready := true.B 556 557 reqs.zipWithIndex.map{ 558 case (req, i) => 559 when (req.valid && canEnqueue) { 560 v(enqPtrVec(i)) := !tlb_req_flushed(i) 561 vpn(enqPtrVec(i)) := req.bits.vpn 562 memidx(enqPtrVec(i)) := req.bits.memidx 563 ports(enqPtrVec(i)) := req_ports(i).asBools 564 } 565 } 566 for (i <- ports.indices) { 567 when (v(i)) { 568 ports(i) := ports(i).zip(update_ports(i)).map(a => a._1 || a._2) 569 } 570 } 571 572 val do_enq = canEnqueue && Cat(reqs.map(_.valid)).orR 573 val do_deq = (!v(deqPtr) && !isEmptyDeq) 574 val do_iss = issue_fire_fake || (!v(issPtr) && !isEmptyIss) 575 when (do_enq) { 576 enqPtr := enqPtr + enqNum 577 } 578 when (do_deq) { 579 deqPtr := deqPtr + 1.U 580 } 581 when (do_iss) { 582 issPtr := issPtr + 1.U 583 } 584 when (issue_fire_fake && issue_filtered) { // issued but is filtered 585 v(issPtr) := false.B 586 } 587 when (do_enq =/= do_deq) { 588 mayFullDeq := do_enq 589 } 590 when (do_enq =/= do_iss) { 591 mayFullIss := do_enq 592 } 593 594 when (io.ptw.resp.fire) { 595 v.zip(ptwResp_OldMatchVec).map{ case (vi, mi) => when (mi) { vi := false.B }} 596 } 597 598 counter := counter - do_deq + Mux(do_enq, enqNum, 0.U) 599 assert(counter <= Size.U, "counter should be no more than Size") 600 assert(inflight_counter <= Size.U, "inflight should be no more than Size") 601 when (counter === 0.U) { 602 assert(!io.ptw.req(0).fire, "when counter is 0, should not req") 603 assert(isEmptyDeq && isEmptyIss, "when counter is 0, should be empty") 604 } 605 when (counter === Size.U) { 606 assert(mayFullDeq, "when counter is Size, should be full") 607 } 608 609 when (flush) { 610 v.map(_ := false.B) 611 deqPtr := 0.U 612 enqPtr := 0.U 613 issPtr := 0.U 614 ptwResp_valid := false.B 615 mayFullDeq := false.B 616 mayFullIss := false.B 617 counter := 0.U 618 inflight_counter := 0.U 619 } 620 621 val robHeadVaddr = io.debugTopDown.robHeadVaddr 622 io.rob_head_miss_in_tlb := VecInit(v.zip(vpn).map{case (vi, vpni) => { 623 vi && robHeadVaddr.valid && vpni === get_pn(robHeadVaddr.bits) 624 }}).asUInt.orR 625 626 // perf 627 XSPerfAccumulate("tlb_req_count", PopCount(Cat(io.tlb.req.map(_.valid)))) 628 XSPerfAccumulate("tlb_req_count_filtered", Mux(do_enq, accumEnqNum(Width - 1), 0.U)) 629 XSPerfAccumulate("ptw_req_count", io.ptw.req(0).fire) 630 XSPerfAccumulate("ptw_req_cycle", inflight_counter) 631 XSPerfAccumulate("tlb_resp_count", io.tlb.resp.fire) 632 XSPerfAccumulate("ptw_resp_count", io.ptw.resp.fire) 633 XSPerfAccumulate("inflight_cycle", !isEmptyDeq) 634 for (i <- 0 until Size + 1) { 635 XSPerfAccumulate(s"counter${i}", counter === i.U) 636 } 637 638 for (i <- 0 until Size) { 639 TimeOutAssert(v(i), timeOutThreshold, s"Filter ${i} doesn't recv resp in time") 640 } 641} 642 643object PTWRepeater { 644 def apply(fenceDelay: Int, 645 tlb: TlbPtwIO, 646 sfence: SfenceBundle, 647 csr: TlbCsrBundle 648 )(implicit p: Parameters) = { 649 val width = tlb.req.size 650 val repeater = Module(new PTWRepeater(width, fenceDelay)) 651 repeater.io.apply(tlb, sfence, csr) 652 repeater 653 } 654 655 def apply(fenceDelay: Int, 656 tlb: TlbPtwIO, 657 ptw: TlbPtwIO, 658 sfence: SfenceBundle, 659 csr: TlbCsrBundle 660 )(implicit p: Parameters) = { 661 val width = tlb.req.size 662 val repeater = Module(new PTWRepeater(width, fenceDelay)) 663 repeater.io.apply(tlb, ptw, sfence, csr) 664 repeater 665 } 666} 667 668object PTWRepeaterNB { 669 def apply(passReady: Boolean, fenceDelay: Int, 670 tlb: TlbPtwIO, 671 sfence: SfenceBundle, 672 csr: TlbCsrBundle 673 )(implicit p: Parameters) = { 674 val width = tlb.req.size 675 val repeater = Module(new PTWRepeaterNB(width, passReady,fenceDelay)) 676 repeater.io.apply(tlb, sfence, csr) 677 repeater 678 } 679 680 def apply(passReady: Boolean, fenceDelay: Int, 681 tlb: TlbPtwIO, 682 ptw: TlbPtwIO, 683 sfence: SfenceBundle, 684 csr: TlbCsrBundle 685 )(implicit p: Parameters) = { 686 val width = tlb.req.size 687 val repeater = Module(new PTWRepeaterNB(width, passReady, fenceDelay)) 688 repeater.io.apply(tlb, ptw, sfence, csr) 689 repeater 690 } 691} 692 693object PTWFilter { 694 def apply(fenceDelay: Int, 695 tlb: VectorTlbPtwIO, 696 ptw: TlbPtwIO, 697 sfence: SfenceBundle, 698 csr: TlbCsrBundle, 699 size: Int 700 )(implicit p: Parameters) = { 701 val width = tlb.req.size 702 val filter = Module(new PTWFilter(width, size, fenceDelay)) 703 filter.io.apply(tlb, ptw, sfence, csr) 704 filter 705 } 706 707 def apply(fenceDelay: Int, 708 tlb: VectorTlbPtwIO, 709 sfence: SfenceBundle, 710 csr: TlbCsrBundle, 711 size: Int 712 )(implicit p: Parameters) = { 713 val width = tlb.req.size 714 val filter = Module(new PTWFilter(width, size, fenceDelay)) 715 filter.io.apply(tlb, sfence, csr) 716 filter 717 } 718} 719 720object PTWNewFilter { 721 def apply(fenceDelay: Int, 722 tlb: VectorTlbPtwIO, 723 ptw: TlbPtwIO, 724 sfence: SfenceBundle, 725 csr: TlbCsrBundle, 726 size: Int 727 )(implicit p: Parameters) = { 728 val width = tlb.req.size 729 val filter = Module(new PTWNewFilter(width, size, fenceDelay)) 730 filter.io.apply(tlb, ptw, sfence, csr) 731 filter 732 } 733 734 def apply(fenceDelay: Int, 735 tlb: VectorTlbPtwIO, 736 sfence: SfenceBundle, 737 csr: TlbCsrBundle, 738 size: Int 739 )(implicit p: Parameters) = { 740 val width = tlb.req.size 741 val filter = Module(new PTWNewFilter(width, size, fenceDelay)) 742 filter.io.apply(tlb, sfence, csr) 743 filter 744 } 745} 746