1/*************************************************************************************** 2* Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences 3* Copyright (c) 2020-2021 Peng Cheng Laboratory 4* 5* XiangShan is licensed under Mulan PSL v2. 6* You can use this software according to the terms and conditions of the Mulan PSL v2. 7* You may obtain a copy of Mulan PSL v2 at: 8* http://license.coscl.org.cn/MulanPSL2 9* 10* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, 11* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, 12* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. 13* 14* See the Mulan PSL v2 for more details. 15***************************************************************************************/ 16 17package xiangshan.cache.mmu 18 19import org.chipsalliance.cde.config.Parameters 20import chisel3._ 21import chisel3.util._ 22import xiangshan._ 23import xiangshan.cache.{HasDCacheParameters, MemoryOpConstants} 24import utils._ 25import utility._ 26import freechips.rocketchip.diplomacy.{LazyModule, LazyModuleImp} 27import freechips.rocketchip.tilelink._ 28 29class PTWReapterIO(Width: Int)(implicit p: Parameters) extends MMUIOBaseBundle { 30 val tlb = Flipped(new TlbPtwIO(Width)) 31 val ptw = new TlbPtwIO 32 33 def apply(tlb: TlbPtwIO, ptw: TlbPtwIO, sfence: SfenceBundle, csr: TlbCsrBundle): Unit = { 34 this.tlb <> tlb 35 this.ptw <> ptw 36 this.sfence <> sfence 37 this.csr <> csr 38 } 39 40 def apply(tlb: TlbPtwIO, sfence: SfenceBundle, csr: TlbCsrBundle): Unit = { 41 this.tlb <> tlb 42 this.sfence <> sfence 43 this.csr <> csr 44 } 45 46} 47 48class PTWRepeater(Width: Int = 1, FenceDelay: Int)(implicit p: Parameters) extends XSModule with HasPtwConst { 49 val io = IO(new PTWReapterIO(Width)) 50 51 val req_in = if (Width == 1) { 52 io.tlb.req(0) 53 } else { 54 val arb = Module(new RRArbiterInit(io.tlb.req(0).bits.cloneType, Width)) 55 arb.io.in <> io.tlb.req 56 arb.io.out 57 } 58 val (tlb, ptw, flush) = (io.tlb, io.ptw, DelayN(io.sfence.valid || io.csr.satp.changed || io.csr.vsatp.changed || io.csr.hgatp.changed, FenceDelay)) 59 val req = RegEnable(req_in.bits, req_in.fire) 60 val resp = RegEnable(ptw.resp.bits, ptw.resp.fire) 61 val haveOne = BoolStopWatch(req_in.fire, tlb.resp.fire || flush) 62 val sent = BoolStopWatch(ptw.req(0).fire, req_in.fire || flush) 63 val recv = BoolStopWatch(ptw.resp.fire && haveOne, req_in.fire || flush) 64 65 req_in.ready := !haveOne 66 ptw.req(0).valid := haveOne && !sent 67 ptw.req(0).bits := req 68 69 tlb.resp.bits := resp 70 tlb.resp.valid := haveOne && recv 71 ptw.resp.ready := !recv 72 73 XSPerfAccumulate("req_count", ptw.req(0).fire) 74 XSPerfAccumulate("tlb_req_cycle", BoolStopWatch(req_in.fire, tlb.resp.fire || flush)) 75 XSPerfAccumulate("ptw_req_cycle", BoolStopWatch(ptw.req(0).fire, ptw.resp.fire || flush)) 76 77 XSDebug(haveOne, p"haveOne:${haveOne} sent:${sent} recv:${recv} sfence:${flush} req:${req} resp:${resp}") 78 XSDebug(req_in.valid || io.tlb.resp.valid, p"tlb: ${tlb}\n") 79 XSDebug(io.ptw.req(0).valid || io.ptw.resp.valid, p"ptw: ${ptw}\n") 80 assert(!RegNext(recv && io.ptw.resp.valid, init = false.B), "re-receive ptw.resp") 81 XSError(io.ptw.req(0).valid && io.ptw.resp.valid && !flush, "ptw repeater recv resp when sending") 82 XSError(io.ptw.resp.valid && (req.vpn =/= io.ptw.resp.bits.s1.entry.tag), "ptw repeater recv resp with wrong tag") 83 XSError(io.ptw.resp.valid && !io.ptw.resp.ready, "ptw repeater's ptw resp back, but not ready") 84 TimeOutAssert(sent && !recv, timeOutThreshold, "Repeater doesn't recv resp in time") 85} 86 87/* dtlb 88 * 89 */ 90 91class PTWRepeaterNB(Width: Int = 1, passReady: Boolean = false, FenceDelay: Int)(implicit p: Parameters) extends XSModule with HasPtwConst { 92 val io = IO(new PTWReapterIO(Width)) 93 94 val req_in = if (Width == 1) { 95 io.tlb.req(0) 96 } else { 97 val arb = Module(new RRArbiterInit(io.tlb.req(0).bits.cloneType, Width)) 98 arb.io.in <> io.tlb.req 99 arb.io.out 100 } 101 val (tlb, ptw, flush) = (io.tlb, io.ptw, DelayN(io.sfence.valid || io.csr.satp.changed || io.csr.vsatp.changed || io.csr.hgatp.changed, FenceDelay)) 102 /* sent: tlb -> repeater -> ptw 103 * recv: ptw -> repeater -> tlb 104 * different from PTWRepeater 105 */ 106 107 // tlb -> repeater -> ptw 108 val req = RegEnable(req_in.bits, req_in.fire) 109 val sent = BoolStopWatch(req_in.fire, ptw.req(0).fire || flush) 110 req_in.ready := !sent || { if (passReady) ptw.req(0).ready else false.B } 111 ptw.req(0).valid := sent 112 ptw.req(0).bits := req 113 114 // ptw -> repeater -> tlb 115 val resp = RegEnable(ptw.resp.bits, ptw.resp.fire) 116 val recv = BoolStopWatch(ptw.resp.fire, tlb.resp.fire || flush) 117 ptw.resp.ready := !recv || { if (passReady) tlb.resp.ready else false.B } 118 tlb.resp.valid := recv 119 tlb.resp.bits := resp 120 121 XSPerfAccumulate("req", req_in.fire) 122 XSPerfAccumulate("resp", tlb.resp.fire) 123 if (!passReady) { 124 XSPerfAccumulate("req_blank", req_in.valid && sent && ptw.req(0).ready) 125 XSPerfAccumulate("resp_blank", ptw.resp.valid && recv && tlb.resp.ready) 126 XSPerfAccumulate("req_blank_ignore_ready", req_in.valid && sent) 127 XSPerfAccumulate("resp_blank_ignore_ready", ptw.resp.valid && recv) 128 } 129 XSDebug(req_in.valid || io.tlb.resp.valid, p"tlb: ${tlb}\n") 130 XSDebug(io.ptw.req(0).valid || io.ptw.resp.valid, p"ptw: ${ptw}\n") 131} 132 133class PTWFilterIO(Width: Int, hasHint: Boolean = false)(implicit p: Parameters) extends MMUIOBaseBundle { 134 val tlb = Flipped(new VectorTlbPtwIO(Width)) 135 val ptw = new TlbPtwIO() 136 val hint = if (hasHint) Some(new TlbHintIO) else None 137 val rob_head_miss_in_tlb = Output(Bool()) 138 val debugTopDown = new Bundle { 139 val robHeadVaddr = Flipped(Valid(UInt(VAddrBits.W))) 140 } 141 142 def apply(tlb: VectorTlbPtwIO, ptw: TlbPtwIO, sfence: SfenceBundle, csr: TlbCsrBundle): Unit = { 143 this.tlb <> tlb 144 this.ptw <> ptw 145 this.sfence <> sfence 146 this.csr <> csr 147 } 148 149 def apply(tlb: VectorTlbPtwIO, sfence: SfenceBundle, csr: TlbCsrBundle): Unit = { 150 this.tlb <> tlb 151 this.sfence <> sfence 152 this.csr <> csr 153 } 154 155} 156 157class PTWFilterEntryIO(Width: Int, hasHint: Boolean = false)(implicit p: Parameters) extends PTWFilterIO(Width, hasHint){ 158 val flush = Input(Bool()) 159 val refill = Output(Bool()) 160 val getGpa = Output(Bool()) 161 val memidx = Output(new MemBlockidxBundle) 162} 163 164class PTWFilterEntry(Width: Int, Size: Int, hasHint: Boolean = false)(implicit p: Parameters) extends XSModule with HasPtwConst { 165 private val LdExuCnt = backendParams.LdExuCnt 166 167 val io = IO(new PTWFilterEntryIO(Width, hasHint)) 168 require(isPow2(Size), s"Filter Size ($Size) must be a power of 2") 169 170 def firstValidIndex(v: Seq[Bool], valid: Bool): UInt = { 171 val index = WireInit(0.U(log2Up(Size).W)) 172 for (i <- 0 until v.size) { 173 when (v(i) === valid) { 174 index := i.U 175 } 176 } 177 index 178 } 179 180 val v = RegInit(VecInit(Seq.fill(Size)(false.B))) 181 val sent = RegInit(VecInit(Seq.fill(Size)(false.B))) 182 val vpn = Reg(Vec(Size, UInt(vpnLen.W))) 183 val s2xlate = Reg(Vec(Size, UInt(2.W))) 184 val getGpa = Reg(Vec(Size, Bool())) 185 val memidx = Reg(Vec(Size, new MemBlockidxBundle)) 186 187 val enqvalid = WireInit(VecInit(Seq.fill(Width)(false.B))) 188 val canenq = WireInit(VecInit(Seq.fill(Width)(false.B))) 189 val enqidx = WireInit(VecInit(Seq.fill(Width)(0.U(log2Up(Size).W)))) 190 191 //val selectCount = RegInit(0.U(log2Up(Width).W)) 192 193 val entryIsMatchVec = WireInit(VecInit(Seq.fill(Width)(false.B))) 194 val entryMatchIndexVec = WireInit(VecInit(Seq.fill(Width)(0.U(log2Up(Size).W)))) 195 val ptwResp_EntryMatchVec = vpn.zip(v).zip(s2xlate).map{ case ((pi, vi), s2xlatei) => vi && s2xlatei === io.ptw.resp.bits.s2xlate && io.ptw.resp.bits.hit(pi, io.csr.satp.asid, io.csr.vsatp.asid, io.csr.hgatp.vmid, true)} 196 val ptwResp_EntryMatchFirst = firstValidIndex(ptwResp_EntryMatchVec, true.B) 197 val ptwResp_ReqMatchVec = io.tlb.req.map(a => io.ptw.resp.valid && a.bits.s2xlate === io.ptw.resp.bits.s2xlate && io.ptw.resp.bits.hit(a.bits.vpn, io.csr.satp.asid, io.csr.vsatp.asid, io.csr.hgatp.vmid, true)) 198 199 io.refill := Cat(ptwResp_EntryMatchVec).orR && io.ptw.resp.fire 200 io.ptw.resp.ready := true.B 201 // DontCare 202 io.tlb.req.map(_.ready := true.B) 203 io.tlb.resp.valid := false.B 204 io.tlb.resp.bits.data := 0.U.asTypeOf(new PtwRespS2withMemIdx) 205 io.tlb.resp.bits.vector := 0.U.asTypeOf(Vec(Width, Bool())) 206 io.tlb.resp.bits.getGpa := 0.U.asTypeOf(Vec(Width, Bool())) 207 io.memidx := 0.U.asTypeOf(new MemBlockidxBundle) 208 io.getGpa := 0.U 209 210 // ugly code, should be optimized later 211 require(Width <= 4, s"DTLB Filter Width ($Width) must equal or less than 4") 212 if (Width == 1) { 213 require(Size == 8, s"prefetch filter Size ($Size) should be 8") 214 canenq(0) := !(Cat(v).andR) 215 enqidx(0) := firstValidIndex(v, false.B) 216 } else if (Width == 2) { 217 require(Size == 8, s"store filter Size ($Size) should be 8") 218 canenq(0) := !(Cat(v.take(Size/2)).andR) 219 enqidx(0) := firstValidIndex(v.take(Size/2), false.B) 220 canenq(1) := !(Cat(v.drop(Size/2)).andR) 221 enqidx(1) := firstValidIndex(v.drop(Size/2), false.B) + (Size/2).U 222 } else if (Width == 3) { 223 require(Size == 16, s"load filter Size ($Size) should be 16") 224 canenq(0) := !(Cat(v.take(8)).andR) 225 enqidx(0) := firstValidIndex(v.take(8), false.B) 226 canenq(1) := !(Cat(v.drop(8).take(4)).andR) 227 enqidx(1) := firstValidIndex(v.drop(8).take(4), false.B) + 8.U 228 // four entries for prefetch 229 canenq(2) := !(Cat(v.drop(12)).andR) 230 enqidx(2) := firstValidIndex(v.drop(12), false.B) + 12.U 231 } else if (Width == 4) { 232 require(Size == 16, s"load filter Size ($Size) should be 16") 233 for (i <- 0 until Width) { 234 canenq(i) := !(Cat(VecInit(v.slice(i * 4, (i + 1) * 4))).andR) 235 enqidx(i) := firstValidIndex(v.slice(i * 4, (i + 1) * 4), false.B) + (i * 4).U 236 } 237 } 238 239 for (i <- 0 until Width) { 240 enqvalid(i) := io.tlb.req(i).valid && !ptwResp_ReqMatchVec(i) && !entryIsMatchVec(i) && canenq(i) 241 when (!enqvalid(i)) { 242 enqidx(i) := entryMatchIndexVec(i) 243 } 244 245 val entryIsMatch = vpn.zip(v).zip(s2xlate).map{ case ((pi, vi), s2xlatei) => vi && s2xlatei === io.tlb.req(i).bits.s2xlate && pi === io.tlb.req(i).bits.vpn} 246 entryIsMatchVec(i) := Cat(entryIsMatch).orR 247 entryMatchIndexVec(i) := firstValidIndex(entryIsMatch, true.B) 248 249 if (i > 0) { 250 for (j <- 0 until i) { 251 val newIsMatch = io.tlb.req(i).bits.vpn === io.tlb.req(j).bits.vpn && io.tlb.req(i).bits.s2xlate === io.tlb.req(j).bits.s2xlate 252 when (newIsMatch && io.tlb.req(j).valid) { 253 enqidx(i) := enqidx(j) 254 canenq(i) := canenq(j) 255 enqvalid(i) := false.B 256 } 257 } 258 } 259 260 when (enqvalid(i)) { 261 v(enqidx(i)) := true.B 262 sent(enqidx(i)) := false.B 263 vpn(enqidx(i)) := io.tlb.req(i).bits.vpn 264 s2xlate(enqidx(i)) := io.tlb.req(i).bits.s2xlate 265 getGpa(enqidx(i)) := io.tlb.req(i).bits.getGpa 266 memidx(enqidx(i)) := io.tlb.req(i).bits.memidx 267 } 268 } 269 270 val issuevec = v.zip(sent).map{ case (v, s) => v && !s} 271 val issueindex = firstValidIndex(issuevec, true.B) 272 val canissue = Cat(issuevec).orR 273 for (i <- 0 until Size) { 274 io.ptw.req(0).valid := canissue 275 io.ptw.req(0).bits.vpn := vpn(issueindex) 276 io.ptw.req(0).bits.s2xlate := s2xlate(issueindex) 277 } 278 when (io.ptw.req(0).fire) { 279 sent(issueindex) := true.B 280 } 281 282 when (io.ptw.resp.fire) { 283 v.zip(ptwResp_EntryMatchVec).map{ case (vi, mi) => when (mi) { vi := false.B }} 284 io.memidx := memidx(ptwResp_EntryMatchFirst) 285 io.getGpa := getGpa(ptwResp_EntryMatchFirst) 286 } 287 288 when (io.flush) { 289 v.map(_ := false.B) 290 } 291 292 if (hasHint) { 293 val hintIO = io.hint.getOrElse(new TlbHintIO) 294 for (i <- 0 until LdExuCnt) { 295 hintIO.req(i).id := enqidx(i) 296 hintIO.req(i).full := !canenq(i) || ptwResp_ReqMatchVec(i) 297 } 298 hintIO.resp.valid := io.refill 299 hintIO.resp.bits.id := ptwResp_EntryMatchFirst 300 hintIO.resp.bits.replay_all := PopCount(ptwResp_EntryMatchVec) > 1.U 301 } 302 303 io.rob_head_miss_in_tlb := VecInit(v.zip(vpn).map{case (vi, vpni) => { 304 vi && io.debugTopDown.robHeadVaddr.valid && vpni === get_pn(io.debugTopDown.robHeadVaddr.bits) 305 }}).asUInt.orR 306 307 308 // Perf Counter 309 val counter = PopCount(v) 310 val inflight_counter = RegInit(0.U(log2Up(Size).W)) 311 val inflight_full = inflight_counter === Size.U 312 when (io.ptw.req(0).fire =/= io.ptw.resp.fire) { 313 inflight_counter := Mux(io.ptw.req(0).fire, inflight_counter + 1.U, inflight_counter - 1.U) 314 } 315 316 assert(inflight_counter <= Size.U, "inflight should be no more than Size") 317 when (counter === 0.U) { 318 assert(!io.ptw.req(0).fire, "when counter is 0, should not req") 319 } 320 321 when (io.flush) { 322 inflight_counter := 0.U 323 } 324 325 XSPerfAccumulate("tlb_req_count", PopCount(Cat(io.tlb.req.map(_.valid)))) 326 XSPerfAccumulate("tlb_req_count_filtered", PopCount(enqvalid)) 327 XSPerfAccumulate("ptw_req_count", io.ptw.req(0).fire) 328 XSPerfAccumulate("ptw_req_cycle", inflight_counter) 329 XSPerfAccumulate("tlb_resp_count", io.tlb.resp.fire) 330 XSPerfAccumulate("ptw_resp_count", io.ptw.resp.fire) 331 XSPerfAccumulate("inflight_cycle", Cat(sent).orR) 332 333 for (i <- 0 until Size + 1) { 334 XSPerfAccumulate(s"counter${i}", counter === i.U) 335 } 336 337 for (i <- 0 until Size) { 338 TimeOutAssert(v(i), timeOutThreshold, s"Filter ${i} doesn't recv resp in time") 339 } 340 341} 342 343class PTWNewFilter(Width: Int, Size: Int, FenceDelay: Int)(implicit p: Parameters) extends XSModule with HasPtwConst { 344 require(Size >= Width) 345 346 private val LduCnt = backendParams.LduCnt 347 private val HyuCnt = backendParams.HyuCnt 348 private val StaCnt = backendParams.StaCnt 349 // all load execute units, including ldu and hyu 350 private val LdExuCnt = backendParams.LdExuCnt 351 // all store address execute units, including sta and hyu 352 private val StaExuCnt = backendParams.StaExuCnt 353 354 val io = IO(new PTWFilterIO(Width, hasHint = true)) 355 356 val load_filter = VecInit(Seq.fill(1) { 357 val load_entry = Module(new PTWFilterEntry(Width = LdExuCnt + 1, Size = loadfiltersize, hasHint = true)) 358 load_entry.io 359 }) 360 361 val store_filter = VecInit(Seq.fill(1) { 362 val store_entry = Module(new PTWFilterEntry(Width = StaCnt, Size = storefiltersize)) 363 store_entry.io 364 }) 365 366 val prefetch_filter = VecInit(Seq.fill(1) { 367 val prefetch_entry = Module(new PTWFilterEntry(Width = 2, Size = prefetchfiltersize)) 368 prefetch_entry.io 369 }) 370 371 val filter = load_filter ++ store_filter ++ prefetch_filter 372 373 load_filter.map(_.tlb.req := io.tlb.req.take(LdExuCnt + 1)) 374 store_filter.map(_.tlb.req := io.tlb.req.drop(LdExuCnt + 1).take(StaCnt)) 375 prefetch_filter.map(_.tlb.req := io.tlb.req.drop(LdExuCnt + 1 + StaCnt)) 376 377 val flush = DelayN(io.sfence.valid || io.csr.satp.changed || (io.csr.priv.virt && io.csr.vsatp.changed), FenceDelay) 378 val ptwResp = RegEnable(io.ptw.resp.bits, io.ptw.resp.fire) 379 val ptwResp_valid = Cat(filter.map(_.refill)).orR 380 filter.map(_.tlb.resp.ready := true.B) 381 filter.map(_.ptw.resp.valid := GatedValidRegNext(io.ptw.resp.fire, init = false.B)) 382 filter.map(_.ptw.resp.bits := ptwResp) 383 filter.map(_.flush := flush) 384 filter.map(_.sfence := io.sfence) 385 filter.map(_.csr := io.csr) 386 filter.map(_.debugTopDown.robHeadVaddr := io.debugTopDown.robHeadVaddr) 387 388 io.tlb.req.map(_.ready := true.B) 389 io.tlb.resp.valid := ptwResp_valid 390 io.tlb.resp.bits.data.s2xlate := ptwResp.s2xlate 391 io.tlb.resp.bits.data.getGpa := DontCare // not used 392 io.tlb.resp.bits.data.s1 := ptwResp.s1 393 io.tlb.resp.bits.data.s2 := ptwResp.s2 394 io.tlb.resp.bits.data.memidx := 0.U.asTypeOf(new MemBlockidxBundle) 395 // vector used to represent different requestors of DTLB 396 // (e.g. the store DTLB has StuCnt requestors) 397 // However, it is only necessary to distinguish between different DTLB now 398 for (i <- 0 until Width) { 399 io.tlb.resp.bits.vector(i) := false.B 400 io.tlb.resp.bits.getGpa(i) := false.B 401 } 402 io.tlb.resp.bits.vector(0) := load_filter(0).refill 403 io.tlb.resp.bits.vector(LdExuCnt + 1) := store_filter(0).refill 404 io.tlb.resp.bits.vector(LdExuCnt + 1 + StaCnt) := prefetch_filter(0).refill 405 io.tlb.resp.bits.getGpa(0) := load_filter(0).getGpa 406 io.tlb.resp.bits.getGpa(LdExuCnt + 1) := store_filter(0).getGpa 407 io.tlb.resp.bits.getGpa(LdExuCnt + 1 + StaCnt) := prefetch_filter(0).getGpa 408 409 val hintIO = io.hint.getOrElse(new TlbHintIO) 410 val load_hintIO = load_filter(0).hint.getOrElse(new TlbHintIO) 411 for (i <- 0 until LdExuCnt) { 412 hintIO.req(i) := RegNext(load_hintIO.req(i)) 413 } 414 hintIO.resp.valid := RegNext(load_hintIO.resp.valid) 415 hintIO.resp.bits := RegEnable(load_hintIO.resp.bits, load_hintIO.resp.valid) 416 417 when (load_filter(0).refill) { 418 io.tlb.resp.bits.vector(0) := true.B 419 io.tlb.resp.bits.data.memidx := load_filter(0).memidx 420 } 421 when (store_filter(0).refill) { 422 io.tlb.resp.bits.vector(LdExuCnt + 1) := true.B 423 io.tlb.resp.bits.data.memidx := store_filter(0).memidx 424 } 425 when (prefetch_filter(0).refill) { 426 io.tlb.resp.bits.vector(LdExuCnt + 1 + StaCnt) := true.B 427 io.tlb.resp.bits.data.memidx := 0.U.asTypeOf(new MemBlockidxBundle) 428 } 429 430 val ptw_arb = Module(new RRArbiterInit(new PtwReq, 3)) 431 for (i <- 0 until 3) { 432 ptw_arb.io.in(i).valid := filter(i).ptw.req(0).valid 433 ptw_arb.io.in(i).bits.vpn := filter(i).ptw.req(0).bits.vpn 434 ptw_arb.io.in(i).bits.s2xlate := filter(i).ptw.req(0).bits.s2xlate 435 filter(i).ptw.req(0).ready := ptw_arb.io.in(i).ready 436 } 437 ptw_arb.io.out.ready := io.ptw.req(0).ready 438 io.ptw.req(0).valid := ptw_arb.io.out.valid 439 io.ptw.req(0).bits.vpn := ptw_arb.io.out.bits.vpn 440 io.ptw.req(0).bits.s2xlate := ptw_arb.io.out.bits.s2xlate 441 io.ptw.resp.ready := true.B 442 443 io.rob_head_miss_in_tlb := Cat(filter.map(_.rob_head_miss_in_tlb)).orR 444} 445 446class PTWFilter(Width: Int, Size: Int, FenceDelay: Int)(implicit p: Parameters) extends XSModule with HasPtwConst { 447 require(Size >= Width) 448 449 val io = IO(new PTWFilterIO(Width)) 450 451 val v = RegInit(VecInit(Seq.fill(Size)(false.B))) 452 val ports = Reg(Vec(Size, Vec(Width, Bool()))) // record which port(s) the entry come from, may not able to cover all the ports 453 val vpn = Reg(Vec(Size, UInt(vpnLen.W))) 454 val s2xlate = Reg(Vec(Size, UInt(2.W))) 455 val getGpa = Reg(Vec(Size, Bool())) 456 val memidx = Reg(Vec(Size, new MemBlockidxBundle)) 457 val enqPtr = RegInit(0.U(log2Up(Size).W)) // Enq 458 val issPtr = RegInit(0.U(log2Up(Size).W)) // Iss to Ptw 459 val deqPtr = RegInit(0.U(log2Up(Size).W)) // Deq 460 val mayFullDeq = RegInit(false.B) 461 val mayFullIss = RegInit(false.B) 462 val counter = RegInit(0.U(log2Up(Size+1).W)) 463 val flush = DelayN(io.sfence.valid || io.csr.satp.changed || io.csr.vsatp.changed || io.csr.hgatp.changed, FenceDelay) 464 val tlb_req = WireInit(io.tlb.req) // NOTE: tlb_req is not io.tlb.req, see below codes, just use cloneType 465 tlb_req.suggestName("tlb_req") 466 467 val inflight_counter = RegInit(0.U(log2Up(Size + 1).W)) 468 val inflight_full = inflight_counter === Size.U 469 470 def ptwResp_hit(vpn: UInt, s2xlate: UInt, resp: PtwRespS2): Bool = { 471 s2xlate === resp.s2xlate && resp.hit(vpn, io.csr.satp.asid, io.csr.vsatp.asid, io.csr.hgatp.vmid, true) 472 } 473 474 when (io.ptw.req(0).fire =/= io.ptw.resp.fire) { 475 inflight_counter := Mux(io.ptw.req(0).fire, inflight_counter + 1.U, inflight_counter - 1.U) 476 } 477 478 val canEnqueue = Wire(Bool()) // NOTE: actually enqueue 479 val ptwResp = RegEnable(io.ptw.resp.bits, io.ptw.resp.fire) 480 val ptwResp_OldMatchVec = vpn.zip(v).zip(s2xlate).map { case (((vpn, v), s2xlate)) =>{ 481 v && ptwResp_hit(vpn, s2xlate, io.ptw.resp.bits) 482 } 483 } 484 val ptwResp_valid = GatedValidRegNext(io.ptw.resp.fire && Cat(ptwResp_OldMatchVec).orR, init = false.B) 485 // May send repeated requests to L2 tlb with same vpn(26, 3) when sector tlb 486 val oldMatchVec_early = io.tlb.req.map(a => vpn.zip(v).zip(s2xlate).map{ case ((pi, vi), s2xlate) => vi && pi === a.bits.vpn && s2xlate === a.bits.s2xlate }) 487 val lastReqMatchVec_early = io.tlb.req.map(a => tlb_req.map{ b => b.valid && b.bits.vpn === a.bits.vpn && canEnqueue && b.bits.s2xlate === a.bits.s2xlate}) 488 val newMatchVec_early = io.tlb.req.map(a => io.tlb.req.map(b => a.bits.vpn === b.bits.vpn && a.bits.s2xlate === b.bits.s2xlate)) 489 490 (0 until Width) foreach { i => 491 tlb_req(i).valid := GatedValidRegNext(io.tlb.req(i).valid && 492 !(ptwResp_valid && ptwResp_hit(io.tlb.req(i).bits.vpn, io.tlb.req(i).bits.s2xlate, ptwResp)) && 493 !Cat(lastReqMatchVec_early(i)).orR, 494 init = false.B) 495 tlb_req(i).bits := RegEnable(io.tlb.req(i).bits, io.tlb.req(i).valid) 496 } 497 498 499 val oldMatchVec = oldMatchVec_early.map(a => GatedValidRegNext(Cat(a).orR)) 500 val newMatchVec = (0 until Width).map(i => (0 until Width).map(j => 501 GatedValidRegNext(newMatchVec_early(i)(j)) && tlb_req(j).valid 502 )) 503 val ptwResp_newMatchVec = tlb_req.map(a => 504 ptwResp_valid && ptwResp_hit(a.bits.vpn, a.bits.s2xlate, ptwResp)) 505 506 val oldMatchVec2 = (0 until Width).map(i => oldMatchVec_early(i).map(GatedValidRegNext(_)).map(_ & tlb_req(i).valid)) 507 val update_ports = v.indices.map(i => oldMatchVec2.map(j => j(i))) 508 val ports_init = (0 until Width).map(i => (1 << i).U(Width.W)) 509 val filter_ports = (0 until Width).map(i => ParallelMux(newMatchVec(i).zip(ports_init).drop(i))) 510 val resp_vector = RegEnable(ParallelMux(ptwResp_OldMatchVec zip ports), io.ptw.resp.fire) 511 val resp_getGpa = RegEnable(ParallelMux(ptwResp_OldMatchVec zip getGpa), io.ptw.resp.fire) 512 513 def canMerge(index: Int) : Bool = { 514 ptwResp_newMatchVec(index) || oldMatchVec(index) || 515 Cat(newMatchVec(index).take(index)).orR 516 } 517 518 def filter_req() = { 519 val reqs = tlb_req.indices.map{ i => 520 val req = Wire(ValidIO(new PtwReqwithMemIdx())) 521 val merge = canMerge(i) 522 req.bits := tlb_req(i).bits 523 req.valid := !merge && tlb_req(i).valid 524 req 525 } 526 reqs 527 } 528 529 val reqs = filter_req() 530 val req_ports = filter_ports 531 val isFull = enqPtr === deqPtr && mayFullDeq 532 val isEmptyDeq = enqPtr === deqPtr && !mayFullDeq 533 val isEmptyIss = enqPtr === issPtr && !mayFullIss 534 val accumEnqNum = (0 until Width).map(i => PopCount(reqs.take(i).map(_.valid))) 535 val enqPtrVecInit = VecInit((0 until Width).map(i => enqPtr + i.U)) 536 val enqPtrVec = VecInit((0 until Width).map(i => enqPtrVecInit(accumEnqNum(i)))) 537 val enqNum = PopCount(reqs.map(_.valid)) 538 canEnqueue := counter +& enqNum <= Size.U 539 540 // the req may recv false ready, but actually received. Filter and TLB will handle it. 541 val enqNum_fake = PopCount(io.tlb.req.map(_.valid)) 542 val canEnqueue_fake = counter +& enqNum_fake <= Size.U 543 io.tlb.req.map(_.ready := canEnqueue_fake) // NOTE: just drop un-fire reqs 544 545 // tlb req flushed by ptw resp: last ptw resp && current ptw resp 546 // the flushed tlb req will fakely enq, with a false valid 547 val tlb_req_flushed = reqs.map(a => io.ptw.resp.valid && ptwResp_hit(a.bits.vpn, a.bits.s2xlate, io.ptw.resp.bits)) 548 549 io.tlb.resp.valid := ptwResp_valid 550 io.tlb.resp.bits.data.s2xlate := ptwResp.s2xlate 551 io.tlb.resp.bits.data.s1 := ptwResp.s1 552 io.tlb.resp.bits.data.s2 := ptwResp.s2 553 io.tlb.resp.bits.data.memidx := memidx(OHToUInt(ptwResp_OldMatchVec)) 554 io.tlb.resp.bits.vector := resp_vector 555 io.tlb.resp.bits.data.getGpa := RegNext(getGpa(OHToUInt(ptwResp_OldMatchVec))) 556 io.tlb.resp.bits.getGpa := DontCare 557 558 val issue_valid = v(issPtr) && !isEmptyIss && !inflight_full 559 val issue_filtered = ptwResp_valid && ptwResp_hit(io.ptw.req(0).bits.vpn, io.ptw.req(0).bits.s2xlate, ptwResp) 560 val issue_fire_fake = issue_valid && (io.ptw.req(0).ready || (issue_filtered && false.B /*timing-opt*/)) 561 io.ptw.req(0).valid := issue_valid && !issue_filtered 562 io.ptw.req(0).bits.vpn := vpn(issPtr) 563 io.ptw.req(0).bits.s2xlate := s2xlate(issPtr) 564 io.ptw.resp.ready := true.B 565 566 reqs.zipWithIndex.map{ 567 case (req, i) => 568 when (req.valid && canEnqueue) { 569 v(enqPtrVec(i)) := !tlb_req_flushed(i) 570 vpn(enqPtrVec(i)) := req.bits.vpn 571 s2xlate(enqPtrVec(i)) := req.bits.s2xlate 572 getGpa(enqPtrVec(i)) := req.bits.getGpa 573 memidx(enqPtrVec(i)) := req.bits.memidx 574 ports(enqPtrVec(i)) := req_ports(i).asBools 575 } 576 } 577 for (i <- ports.indices) { 578 when (v(i)) { 579 ports(i) := ports(i).zip(update_ports(i)).map(a => a._1 || a._2) 580 } 581 } 582 583 val do_enq = canEnqueue && Cat(reqs.map(_.valid)).orR 584 val do_deq = (!v(deqPtr) && !isEmptyDeq) 585 val do_iss = issue_fire_fake || (!v(issPtr) && !isEmptyIss) 586 when (do_enq) { 587 enqPtr := enqPtr + enqNum 588 } 589 when (do_deq) { 590 deqPtr := deqPtr + 1.U 591 } 592 when (do_iss) { 593 issPtr := issPtr + 1.U 594 } 595 when (issue_fire_fake && issue_filtered) { // issued but is filtered 596 v(issPtr) := false.B 597 } 598 when (do_enq =/= do_deq) { 599 mayFullDeq := do_enq 600 } 601 when (do_enq =/= do_iss) { 602 mayFullIss := do_enq 603 } 604 605 when (io.ptw.resp.fire) { 606 v.zip(ptwResp_OldMatchVec).map{ case (vi, mi) => when (mi) { vi := false.B }} 607 } 608 609 counter := counter - do_deq + Mux(do_enq, enqNum, 0.U) 610 assert(counter <= Size.U, "counter should be no more than Size") 611 assert(inflight_counter <= Size.U, "inflight should be no more than Size") 612 when (counter === 0.U) { 613 assert(!io.ptw.req(0).fire, "when counter is 0, should not req") 614 assert(isEmptyDeq && isEmptyIss, "when counter is 0, should be empty") 615 } 616 when (counter === Size.U) { 617 assert(mayFullDeq, "when counter is Size, should be full") 618 } 619 620 when (flush) { 621 v.map(_ := false.B) 622 deqPtr := 0.U 623 enqPtr := 0.U 624 issPtr := 0.U 625 ptwResp_valid := false.B 626 mayFullDeq := false.B 627 mayFullIss := false.B 628 counter := 0.U 629 inflight_counter := 0.U 630 } 631 632 val robHeadVaddr = io.debugTopDown.robHeadVaddr 633 io.rob_head_miss_in_tlb := VecInit(v.zip(vpn).map{case (vi, vpni) => { 634 vi && robHeadVaddr.valid && vpni === get_pn(robHeadVaddr.bits) 635 }}).asUInt.orR 636 637 // perf 638 XSPerfAccumulate("tlb_req_count", PopCount(Cat(io.tlb.req.map(_.valid)))) 639 XSPerfAccumulate("tlb_req_count_filtered", Mux(do_enq, accumEnqNum(Width - 1), 0.U)) 640 XSPerfAccumulate("ptw_req_count", io.ptw.req(0).fire) 641 XSPerfAccumulate("ptw_req_cycle", inflight_counter) 642 XSPerfAccumulate("tlb_resp_count", io.tlb.resp.fire) 643 XSPerfAccumulate("ptw_resp_count", io.ptw.resp.fire) 644 XSPerfAccumulate("inflight_cycle", !isEmptyDeq) 645 for (i <- 0 until Size + 1) { 646 XSPerfAccumulate(s"counter${i}", counter === i.U) 647 } 648 649 for (i <- 0 until Size) { 650 TimeOutAssert(v(i), timeOutThreshold, s"Filter ${i} doesn't recv resp in time") 651 } 652} 653 654object PTWRepeater { 655 def apply(fenceDelay: Int, 656 tlb: TlbPtwIO, 657 sfence: SfenceBundle, 658 csr: TlbCsrBundle 659 )(implicit p: Parameters) = { 660 val width = tlb.req.size 661 val repeater = Module(new PTWRepeater(width, fenceDelay)) 662 repeater.io.apply(tlb, sfence, csr) 663 repeater 664 } 665 666 def apply(fenceDelay: Int, 667 tlb: TlbPtwIO, 668 ptw: TlbPtwIO, 669 sfence: SfenceBundle, 670 csr: TlbCsrBundle 671 )(implicit p: Parameters) = { 672 val width = tlb.req.size 673 val repeater = Module(new PTWRepeater(width, fenceDelay)) 674 repeater.io.apply(tlb, ptw, sfence, csr) 675 repeater 676 } 677} 678 679object PTWRepeaterNB { 680 def apply(passReady: Boolean, fenceDelay: Int, 681 tlb: TlbPtwIO, 682 sfence: SfenceBundle, 683 csr: TlbCsrBundle 684 )(implicit p: Parameters) = { 685 val width = tlb.req.size 686 val repeater = Module(new PTWRepeaterNB(width, passReady,fenceDelay)) 687 repeater.io.apply(tlb, sfence, csr) 688 repeater 689 } 690 691 def apply(passReady: Boolean, fenceDelay: Int, 692 tlb: TlbPtwIO, 693 ptw: TlbPtwIO, 694 sfence: SfenceBundle, 695 csr: TlbCsrBundle 696 )(implicit p: Parameters) = { 697 val width = tlb.req.size 698 val repeater = Module(new PTWRepeaterNB(width, passReady, fenceDelay)) 699 repeater.io.apply(tlb, ptw, sfence, csr) 700 repeater 701 } 702} 703 704object PTWFilter { 705 def apply(fenceDelay: Int, 706 tlb: VectorTlbPtwIO, 707 ptw: TlbPtwIO, 708 sfence: SfenceBundle, 709 csr: TlbCsrBundle, 710 size: Int 711 )(implicit p: Parameters) = { 712 val width = tlb.req.size 713 val filter = Module(new PTWFilter(width, size, fenceDelay)) 714 filter.io.apply(tlb, ptw, sfence, csr) 715 filter 716 } 717 718 def apply(fenceDelay: Int, 719 tlb: VectorTlbPtwIO, 720 sfence: SfenceBundle, 721 csr: TlbCsrBundle, 722 size: Int 723 )(implicit p: Parameters) = { 724 val width = tlb.req.size 725 val filter = Module(new PTWFilter(width, size, fenceDelay)) 726 filter.io.apply(tlb, sfence, csr) 727 filter 728 } 729} 730 731object PTWNewFilter { 732 def apply(fenceDelay: Int, 733 tlb: VectorTlbPtwIO, 734 ptw: TlbPtwIO, 735 sfence: SfenceBundle, 736 csr: TlbCsrBundle, 737 size: Int 738 )(implicit p: Parameters) = { 739 val width = tlb.req.size 740 val filter = Module(new PTWNewFilter(width, size, fenceDelay)) 741 filter.io.apply(tlb, ptw, sfence, csr) 742 filter 743 } 744 745 def apply(fenceDelay: Int, 746 tlb: VectorTlbPtwIO, 747 sfence: SfenceBundle, 748 csr: TlbCsrBundle, 749 size: Int 750 )(implicit p: Parameters) = { 751 val width = tlb.req.size 752 val filter = Module(new PTWNewFilter(width, size, fenceDelay)) 753 filter.io.apply(tlb, sfence, csr) 754 filter 755 } 756} 757