1/*************************************************************************************** 2* Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences 3* Copyright (c) 2020-2021 Peng Cheng Laboratory 4* 5* XiangShan is licensed under Mulan PSL v2. 6* You can use this software according to the terms and conditions of the Mulan PSL v2. 7* You may obtain a copy of Mulan PSL v2 at: 8* http://license.coscl.org.cn/MulanPSL2 9* 10* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, 11* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, 12* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. 13* 14* See the Mulan PSL v2 for more details. 15***************************************************************************************/ 16 17package xiangshan.cache.mmu 18 19import chipsalliance.rocketchip.config.Parameters 20import chisel3._ 21import chisel3.util._ 22import xiangshan._ 23import xiangshan.cache.{HasDCacheParameters, MemoryOpConstants} 24import utils._ 25import freechips.rocketchip.diplomacy.{LazyModule, LazyModuleImp} 26import freechips.rocketchip.tilelink._ 27 28class PTWReapterIO(Width: Int)(implicit p: Parameters) extends MMUIOBaseBundle { 29 val tlb = Flipped(new TlbPtwIO(Width)) 30 val ptw = new TlbPtwIO 31 32 def apply(tlb: TlbPtwIO, ptw: TlbPtwIO, sfence: SfenceBundle, csr: TlbCsrBundle): Unit = { 33 this.tlb <> tlb 34 this.ptw <> ptw 35 this.sfence <> sfence 36 this.csr <> csr 37 } 38 39 def apply(tlb: TlbPtwIO, sfence: SfenceBundle, csr: TlbCsrBundle): Unit = { 40 this.tlb <> tlb 41 this.sfence <> sfence 42 this.csr <> csr 43 } 44 45 override def cloneType: this.type = (new PTWReapterIO(Width)).asInstanceOf[this.type] 46} 47 48class PTWRepeater(Width: Int = 1)(implicit p: Parameters) extends XSModule with HasPtwConst { 49 val io = IO(new PTWReapterIO(Width)) 50 51 val req_in = if (Width == 1) { 52 io.tlb.req(0) 53 } else { 54 val arb = Module(new RRArbiter(io.tlb.req(0).bits.cloneType, Width)) 55 arb.io.in <> io.tlb.req 56 arb.io.out 57 } 58 val (tlb, ptw, flush) = (io.tlb, io.ptw, RegNext(io.sfence.valid || io.csr.satp.changed)) 59 val req = RegEnable(req_in.bits, req_in.fire()) 60 val resp = RegEnable(ptw.resp.bits, ptw.resp.fire()) 61 val haveOne = BoolStopWatch(req_in.fire(), tlb.resp.fire() || flush) 62 val sent = BoolStopWatch(ptw.req(0).fire(), req_in.fire() || flush) 63 val recv = BoolStopWatch(ptw.resp.fire(), req_in.fire() || flush) 64 65 req_in.ready := !haveOne 66 ptw.req(0).valid := haveOne && !sent 67 ptw.req(0).bits := req 68 69 tlb.resp.bits := resp 70 tlb.resp.valid := haveOne && recv 71 ptw.resp.ready := !recv 72 73 XSPerfAccumulate("req_count", ptw.req(0).fire()) 74 XSPerfAccumulate("tlb_req_cycle", BoolStopWatch(req_in.fire(), tlb.resp.fire() || flush)) 75 XSPerfAccumulate("ptw_req_cycle", BoolStopWatch(ptw.req(0).fire(), ptw.resp.fire() || flush)) 76 77 XSDebug(haveOne, p"haveOne:${haveOne} sent:${sent} recv:${recv} sfence:${flush} req:${req} resp:${resp}") 78 XSDebug(req_in.valid || io.tlb.resp.valid, p"tlb: ${tlb}\n") 79 XSDebug(io.ptw.req(0).valid || io.ptw.resp.valid, p"ptw: ${ptw}\n") 80 assert(!RegNext(recv && io.ptw.resp.valid, init = false.B), "re-receive ptw.resp") 81 TimeOutAssert(sent && !recv, timeOutThreshold, "Repeater doesn't recv resp in time") 82} 83 84/* dtlb 85 * 86 */ 87 88class PTWRepeaterNB(Width: Int = 1, passReady: Boolean = false)(implicit p: Parameters) extends XSModule with HasPtwConst { 89 val io = IO(new PTWReapterIO(Width)) 90 91 val req_in = if (Width == 1) { 92 io.tlb.req(0) 93 } else { 94 val arb = Module(new RRArbiter(io.tlb.req(0).bits.cloneType, Width)) 95 arb.io.in <> io.tlb.req 96 arb.io.out 97 } 98 val (tlb, ptw, flush) = (io.tlb, io.ptw, RegNext(io.sfence.valid || io.csr.satp.changed)) 99 /* sent: tlb -> repeater -> ptw 100 * recv: ptw -> repeater -> tlb 101 * different from PTWRepeater 102 */ 103 104 // tlb -> repeater -> ptw 105 val req = RegEnable(req_in.bits, req_in.fire()) 106 val sent = BoolStopWatch(req_in.fire(), ptw.req(0).fire() || flush) 107 req_in.ready := !sent || { if (passReady) ptw.req(0).ready else false.B } 108 ptw.req(0).valid := sent 109 ptw.req(0).bits := req 110 111 // ptw -> repeater -> tlb 112 val resp = RegEnable(ptw.resp.bits, ptw.resp.fire()) 113 val recv = BoolStopWatch(ptw.resp.fire(), tlb.resp.fire() || flush) 114 ptw.resp.ready := !recv || { if (passReady) tlb.resp.ready else false.B } 115 tlb.resp.valid := recv 116 tlb.resp.bits := resp 117 118 XSPerfAccumulate("req", req_in.fire()) 119 XSPerfAccumulate("resp", tlb.resp.fire()) 120 if (!passReady) { 121 XSPerfAccumulate("req_blank", req_in.valid && sent && ptw.req(0).ready) 122 XSPerfAccumulate("resp_blank", ptw.resp.valid && recv && tlb.resp.ready) 123 XSPerfAccumulate("req_blank_ignore_ready", req_in.valid && sent) 124 XSPerfAccumulate("resp_blank_ignore_ready", ptw.resp.valid && recv) 125 } 126 XSDebug(req_in.valid || io.tlb.resp.valid, p"tlb: ${tlb}\n") 127 XSDebug(io.ptw.req(0).valid || io.ptw.resp.valid, p"ptw: ${ptw}\n") 128} 129 130class PTWFilterIO(Width: Int)(implicit p: Parameters) extends MMUIOBaseBundle { 131 val tlb = Flipped(new BTlbPtwIO(Width)) 132 val ptw = new TlbPtwIO() 133 134 def apply(tlb: BTlbPtwIO, ptw: TlbPtwIO, sfence: SfenceBundle, csr: TlbCsrBundle): Unit = { 135 this.tlb <> tlb 136 this.ptw <> ptw 137 this.sfence <> sfence 138 this.csr <> csr 139 } 140 141 def apply(tlb: BTlbPtwIO, sfence: SfenceBundle, csr: TlbCsrBundle): Unit = { 142 this.tlb <> tlb 143 this.sfence <> sfence 144 this.csr <> csr 145 } 146 147 override def cloneType: this.type = (new PTWFilterIO(Width)).asInstanceOf[this.type] 148} 149 150class PTWFilter(Width: Int, Size: Int)(implicit p: Parameters) extends XSModule with HasPtwConst { 151 require(Size >= Width) 152 153 val io = IO(new PTWFilterIO(Width)) 154 155 val v = RegInit(VecInit(Seq.fill(Size)(false.B))) 156 val ports = Reg(Vec(Size, Vec(Width, Bool()))) // record which port(s) the entry come from, may not able to cover all the ports 157 val vpn = Reg(Vec(Size, UInt(vpnLen.W))) 158 val enqPtr = RegInit(0.U(log2Up(Size).W)) // Enq 159 val issPtr = RegInit(0.U(log2Up(Size).W)) // Iss to Ptw 160 val deqPtr = RegInit(0.U(log2Up(Size).W)) // Deq 161 val mayFullDeq = RegInit(false.B) 162 val mayFullIss = RegInit(false.B) 163 val counter = RegInit(0.U(log2Up(Size+1).W)) 164 165 val flush = RegNext(io.sfence.valid || io.csr.satp.changed) 166 val ptwResp = RegEnable(io.ptw.resp.bits, io.ptw.resp.fire()) 167 val ptwResp_valid = RegNext(io.ptw.resp.valid, init = false.B) 168 val tlb_req = io.tlb.req 169 val oldMatchVec = tlb_req.map(a => vpn.zip(v).map{case (pi, vi) => vi && a.valid && pi === a.bits.vpn }) 170 val newMatchVec = tlb_req.map(a => tlb_req.map(b => b.valid && a.valid && b.bits.vpn === a.bits.vpn )) 171 val ptwResp_newMatchVec = tlb_req.map(a => ptwResp_valid && ptwResp.entry.hit(a.bits.vpn, io.csr.satp.asid, allType = true) && a.valid) // TODO: may have long latency 172 val ptwResp_oldMatchVec = vpn.zip(v).map{ case (pi, vi) => vi && ptwResp.entry.hit(pi, io.csr.satp.asid, allType = true) } 173 val update_ports = v.indices.map(i => oldMatchVec.map(j => j(i))) 174 val ports_init = (0 until Width).map(i => (1 << i).U(Width.W)) 175 val filter_ports = (0 until Width).map(i => ParallelMux(newMatchVec(i).zip(ports_init).drop(i))) 176 val resp_vector = ParallelMux(ptwResp_oldMatchVec zip ports) 177 val resp_still_valid = ParallelOR(ptwResp_oldMatchVec).asBool 178 179 def canMerge(index: Int) : Bool = { 180 ptwResp_newMatchVec(index) || 181 Cat(oldMatchVec(index)).orR || 182 Cat(newMatchVec(index).take(index)).orR 183 } 184 185 def filter_req() = { 186 val reqs = tlb_req.indices.map{ i => 187 val req = Wire(ValidIO(new PtwReq())) 188 val merge = canMerge(i) 189 req.bits := tlb_req(i).bits 190 req.valid := !merge && tlb_req(i).valid 191 req 192 } 193 reqs 194 } 195 196 val reqs = filter_req() 197 val req_ports = filter_ports 198 var enqPtr_next = WireInit(deqPtr) 199 val isFull = enqPtr === deqPtr && mayFullDeq 200 val isEmptyDeq = enqPtr === deqPtr && !mayFullDeq 201 val isEmptyIss = enqPtr === issPtr && !mayFullIss 202 val accumEnqNum = (0 until Width).map(i => PopCount(reqs.take(i).map(_.valid))) 203 val enqPtrVec = VecInit((0 until Width).map(i => enqPtr + accumEnqNum(i))) 204 val enqNum = PopCount(reqs.map(_.valid)) 205 val canEnqueue = counter +& enqNum <= Size.U 206 207 io.tlb.req.map(_.ready := true.B) // NOTE: just drop un-fire reqs 208 io.tlb.resp.valid := ptwResp_valid && resp_still_valid 209 io.tlb.resp.bits.data := ptwResp 210 io.tlb.resp.bits.vector := resp_vector 211 io.ptw.req(0).valid := v(issPtr) && !isEmptyIss && !(ptwResp_valid && ptwResp.entry.hit(io.ptw.req(0).bits.vpn, io.csr.satp.asid, ignoreAsid = true)) 212 io.ptw.req(0).bits.vpn := vpn(issPtr) 213 io.ptw.resp.ready := true.B 214 215 reqs.zipWithIndex.map{ 216 case (req, i) => 217 when (req.valid && canEnqueue) { 218 v(enqPtrVec(i)) := true.B 219 vpn(enqPtrVec(i)) := req.bits.vpn 220 ports(enqPtrVec(i)) := req_ports(i).asBools 221 } 222 } 223 for (i <- ports.indices) { 224 when (v(i)) { 225 ports(i) := ports(i).zip(update_ports(i)).map(a => a._1 || a._2) 226 } 227 } 228 229 val do_enq = canEnqueue && Cat(reqs.map(_.valid)).orR 230 val do_deq = (!v(deqPtr) && !isEmptyDeq) 231 val do_iss = io.ptw.req(0).fire() || (!v(issPtr) && !isEmptyIss) 232 when (do_enq) { 233 enqPtr := enqPtr + enqNum 234 } 235 when (do_deq) { 236 deqPtr := deqPtr + 1.U 237 } 238 when (do_iss) { 239 issPtr := issPtr + 1.U 240 } 241 when (do_enq =/= do_deq) { 242 mayFullDeq := do_enq 243 } 244 when (do_enq =/= do_iss) { 245 mayFullIss := do_enq 246 } 247 248 when (ptwResp_valid) { 249 v.zip(ptwResp_oldMatchVec).map{ case (vi, mi) => when (mi) { vi := false.B }} 250 } 251 252 counter := counter - do_deq + Mux(do_enq, enqNum, 0.U) 253 assert(counter <= Size.U, "counter should be less than Size") 254 when (counter === 0.U) { 255 assert(!io.ptw.req(0).fire(), "when counter is 0, should not req") 256 assert(isEmptyDeq && isEmptyIss, "when counter is 0, should be empty") 257 } 258 when (counter === Size.U) { 259 assert(mayFullDeq, "when counter is Size, should be full") 260 } 261 262 when (flush) { 263 v.map(_ := false.B) 264 deqPtr := 0.U 265 enqPtr := 0.U 266 issPtr := 0.U 267 ptwResp_valid := false.B 268 mayFullDeq := false.B 269 mayFullIss := false.B 270 counter := 0.U 271 } 272 273 // perf 274 val inflight_counter = RegInit(0.U(log2Up(Size + 1).W)) 275 when (io.ptw.req(0).fire() =/= io.ptw.resp.fire()) { 276 inflight_counter := Mux(io.ptw.req(0).fire(), inflight_counter + 1.U, inflight_counter - 1.U) 277 } 278 when (flush) { 279 inflight_counter := 0.U 280 } 281 XSPerfAccumulate("tlb_req_count", PopCount(Cat(io.tlb.req.map(_.valid)))) 282 XSPerfAccumulate("tlb_req_count_filtered", Mux(do_enq, accumEnqNum(Width - 1), 0.U)) 283 XSPerfAccumulate("ptw_req_count", io.ptw.req(0).fire()) 284 XSPerfAccumulate("ptw_req_cycle", inflight_counter) 285 XSPerfAccumulate("tlb_resp_count", io.tlb.resp.fire()) 286 XSPerfAccumulate("ptw_resp_count", io.ptw.resp.fire()) 287 XSPerfAccumulate("inflight_cycle", !isEmptyDeq) 288 for (i <- 0 until Size + 1) { 289 XSPerfAccumulate(s"counter${i}", counter === i.U) 290 } 291 292 for (i <- 0 until Size) { 293 TimeOutAssert(v(i), timeOutThreshold, s"Filter ${i} doesn't recv resp in time") 294 } 295} 296 297object PTWRepeater { 298 def apply( 299 tlb: TlbPtwIO, 300 sfence: SfenceBundle, 301 csr: TlbCsrBundle 302 )(implicit p: Parameters) = { 303 val width = tlb.req.size 304 val repeater = Module(new PTWRepeater(width)) 305 repeater.io.apply(tlb, sfence, csr) 306 repeater 307 } 308 309 def apply( 310 tlb: TlbPtwIO, 311 ptw: TlbPtwIO, 312 sfence: SfenceBundle, 313 csr: TlbCsrBundle 314 )(implicit p: Parameters) = { 315 val width = tlb.req.size 316 val repeater = Module(new PTWRepeater(width)) 317 repeater.io.apply(tlb, ptw, sfence, csr) 318 repeater 319 } 320} 321 322object PTWRepeaterNB { 323 def apply(passReady: Boolean, 324 tlb: TlbPtwIO, 325 sfence: SfenceBundle, 326 csr: TlbCsrBundle 327 )(implicit p: Parameters) = { 328 val width = tlb.req.size 329 val repeater = Module(new PTWRepeaterNB(width, passReady)) 330 repeater.io.apply(tlb, sfence, csr) 331 repeater 332 } 333 334 def apply(passReady: Boolean, 335 tlb: TlbPtwIO, 336 ptw: TlbPtwIO, 337 sfence: SfenceBundle, 338 csr: TlbCsrBundle 339 )(implicit p: Parameters) = { 340 val width = tlb.req.size 341 val repeater = Module(new PTWRepeaterNB(width, passReady)) 342 repeater.io.apply(tlb, ptw, sfence, csr) 343 repeater 344 } 345} 346 347object PTWFilter { 348 def apply( 349 tlb: BTlbPtwIO, 350 ptw: TlbPtwIO, 351 sfence: SfenceBundle, 352 csr: TlbCsrBundle, 353 size: Int 354 )(implicit p: Parameters) = { 355 val width = tlb.req.size 356 val filter = Module(new PTWFilter(width, size)) 357 filter.io.apply(tlb, ptw, sfence, csr) 358 filter 359 } 360 361 def apply( 362 tlb: BTlbPtwIO, 363 sfence: SfenceBundle, 364 csr: TlbCsrBundle, 365 size: Int 366 )(implicit p: Parameters) = { 367 val width = tlb.req.size 368 val filter = Module(new PTWFilter(width, size)) 369 filter.io.apply(tlb, sfence, csr) 370 filter 371 } 372 373}