1/*************************************************************************************** 2* Copyright (c) 2024 Beijing Institute of Open Source Chip (BOSC) 3* Copyright (c) 2020-2024 Institute of Computing Technology, Chinese Academy of Sciences 4* Copyright (c) 2020-2021 Peng Cheng Laboratory 5* 6* XiangShan is licensed under Mulan PSL v2. 7* You can use this software according to the terms and conditions of the Mulan PSL v2. 8* You may obtain a copy of Mulan PSL v2 at: 9* http://license.coscl.org.cn/MulanPSL2 10* 11* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, 12* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, 13* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. 14* 15* See the Mulan PSL v2 for more details. 16***************************************************************************************/ 17 18package xiangshan.cache.mmu 19 20import org.chipsalliance.cde.config.Parameters 21import chisel3._ 22import chisel3.util._ 23import xiangshan._ 24import xiangshan.cache.{HasDCacheParameters, MemoryOpConstants} 25import utils._ 26import utility._ 27import coupledL2.utils.SplittedSRAM 28import freechips.rocketchip.diplomacy.{LazyModule, LazyModuleImp} 29import freechips.rocketchip.tilelink._ 30 31/* ptw cache caches the page table of all the three layers 32 * ptw cache resp at next cycle 33 * the cache should not be blocked 34 * when miss queue if full, just block req outside 35 */ 36 37class PageCachePerPespBundle(implicit p: Parameters) extends PtwBundle { 38 val hit = Bool() 39 val pre = Bool() 40 val ppn = UInt(gvpnLen.W) 41 val pbmt = UInt(ptePbmtLen.W) 42 val perm = new PtePermBundle() 43 val ecc = Bool() 44 val level = UInt(2.W) 45 val v = Bool() 46 47 def apply(hit: Bool, pre: Bool, ppn: UInt, pbmt: UInt = 0.U, 48 perm: PtePermBundle = 0.U.asTypeOf(new PtePermBundle()), 49 ecc: Bool = false.B, level: UInt = 0.U, valid: Bool = true.B): Unit = { 50 this.hit := hit && !ecc 51 this.pre := pre 52 this.ppn := ppn 53 this.pbmt := pbmt 54 this.perm := perm 55 this.ecc := ecc && hit 56 this.level := level 57 this.v := valid 58 } 59} 60 61class PageCacheMergePespBundle(implicit p: Parameters) extends PtwBundle { 62 assert(tlbcontiguous == 8, "Only support tlbcontiguous = 8!") 63 val hit = Bool() 64 val pre = Bool() 65 val ppn = Vec(tlbcontiguous, UInt(gvpnLen.W)) 66 val pbmt = Vec(tlbcontiguous, UInt(ptePbmtLen.W)) 67 val perm = Vec(tlbcontiguous, new PtePermBundle()) 68 val ecc = Bool() 69 val level = UInt(2.W) 70 val v = Vec(tlbcontiguous, Bool()) 71 72 def apply(hit: Bool, pre: Bool, ppn: Vec[UInt], pbmt: Vec[UInt] = Vec(tlbcontiguous, 0.U), 73 perm: Vec[PtePermBundle] = Vec(tlbcontiguous, 0.U.asTypeOf(new PtePermBundle())), 74 ecc: Bool = false.B, level: UInt = 0.U, valid: Vec[Bool] = Vec(tlbcontiguous, true.B)): Unit = { 75 this.hit := hit && !ecc 76 this.pre := pre 77 this.ppn := ppn 78 this.pbmt := pbmt 79 this.perm := perm 80 this.ecc := ecc && hit 81 this.level := level 82 this.v := valid 83 } 84} 85 86class PageCacheRespBundle(implicit p: Parameters) extends PtwBundle { 87 val l3 = if (EnableSv48) Some(new PageCachePerPespBundle) else None 88 val l2 = new PageCachePerPespBundle 89 val l1 = new PageCachePerPespBundle 90 val l0 = new PageCacheMergePespBundle 91 val sp = new PageCachePerPespBundle 92} 93 94class PtwCacheReq(implicit p: Parameters) extends PtwBundle { 95 val req_info = new L2TlbInnerBundle() 96 val isFirst = Bool() 97 val bypassed = if (EnableSv48) Vec(4, Bool()) else Vec(3, Bool()) 98 val isHptwReq = Bool() 99 val hptwId = UInt(log2Up(l2tlbParams.llptwsize).W) 100} 101 102class PtwCacheIO()(implicit p: Parameters) extends MMUIOBaseBundle with HasPtwConst { 103 val req = Flipped(DecoupledIO(new PtwCacheReq())) 104 val resp = DecoupledIO(new Bundle { 105 val req_info = new L2TlbInnerBundle() 106 val isFirst = Bool() 107 val hit = Bool() 108 val prefetch = Bool() // is the entry fetched by prefetch 109 val bypassed = Bool() 110 val toFsm = new Bundle { 111 val l3Hit = if (EnableSv48) Some(Bool()) else None 112 val l2Hit = Bool() 113 val l1Hit = Bool() 114 val ppn = UInt(gvpnLen.W) 115 val stage1Hit = Bool() // find stage 1 pte in cache, but need to search stage 2 pte in cache at PTW 116 } 117 val stage1 = new PtwMergeResp() 118 val isHptwReq = Bool() 119 val toHptw = new Bundle { 120 val l3Hit = if (EnableSv48) Some(Bool()) else None 121 val l2Hit = Bool() 122 val l1Hit = Bool() 123 val ppn = UInt(ppnLen.W) 124 val id = UInt(log2Up(l2tlbParams.llptwsize).W) 125 val resp = new HptwResp() // used if hit 126 val bypassed = Bool() 127 } 128 }) 129 val refill = Flipped(ValidIO(new Bundle { 130 val ptes = UInt(blockBits.W) 131 val levelOH = new Bundle { 132 // NOTE: levelOH has (Level+1) bits, each stands for page cache entries 133 val sp = Bool() 134 val l0 = Bool() 135 val l1 = Bool() 136 val l2 = Bool() 137 val l3 = if (EnableSv48) Some(Bool()) else None 138 def apply(levelUInt: UInt, valid: Bool) = { 139 sp := GatedValidRegNext((levelUInt === 1.U || levelUInt === 2.U || levelUInt === 3.U) && valid, false.B) 140 l0 := GatedValidRegNext((levelUInt === 0.U) & valid, false.B) 141 l1 := GatedValidRegNext((levelUInt === 1.U) & valid, false.B) 142 l2 := GatedValidRegNext((levelUInt === 2.U) & valid, false.B) 143 l3.map(_ := GatedValidRegNext((levelUInt === 3.U) & valid, false.B)) 144 } 145 } 146 // duplicate level and sel_pte for each page caches, for better fanout 147 val req_info_dup = Vec(3, new L2TlbInnerBundle()) 148 val level_dup = Vec(3, UInt(log2Up(Level + 1).W)) 149 val sel_pte_dup = Vec(3, UInt(XLEN.W)) 150 })) 151 val sfence_dup = Vec(4, Input(new SfenceBundle())) 152 val csr_dup = Vec(3, Input(new TlbCsrBundle())) 153} 154 155class PtwCache()(implicit p: Parameters) extends XSModule with HasPtwConst with HasPerfEvents { 156 val io = IO(new PtwCacheIO) 157 val ecc = Code.fromString(l2tlbParams.ecc) 158 val l1EntryType = new PTWEntriesWithEcc(ecc, num = PtwL1SectorSize, tagLen = PtwL1TagLen, level = 1, hasPerm = false, ReservedBits = l2tlbParams.l1ReservedBits) 159 val l0EntryType = new PTWEntriesWithEcc(ecc, num = PtwL0SectorSize, tagLen = PtwL0TagLen, level = 0, hasPerm = true, ReservedBits = l2tlbParams.l0ReservedBits) 160 161 // TODO: four caches make the codes dirty, think about how to deal with it 162 163 val sfence_dup = io.sfence_dup 164 val refill = io.refill.bits 165 val refill_prefetch_dup = io.refill.bits.req_info_dup.map(a => from_pre(a.source)) 166 val refill_h = io.refill.bits.req_info_dup.map(a => Mux(a.s2xlate === allStage, onlyStage1, a.s2xlate)) 167 val flush_dup = sfence_dup.zip(io.csr_dup).map(f => f._1.valid || f._2.satp.changed || f._2.vsatp.changed || f._2.hgatp.changed) 168 val flush = flush_dup(0) 169 170 // when refill, refuce to accept new req 171 val rwHarzad = if (sramSinglePort) io.refill.valid else false.B 172 173 // handle hand signal and req_info 174 // TODO: replace with FlushableQueue 175 val stageReq = Wire(Decoupled(new PtwCacheReq())) // enq stage & read page cache valid 176 val stageDelay = Wire(Vec(2, Decoupled(new PtwCacheReq()))) // page cache resp 177 val stageCheck = Wire(Vec(2, Decoupled(new PtwCacheReq()))) // check hit & check ecc 178 val stageResp = Wire(Decoupled(new PtwCacheReq())) // deq stage 179 180 val stageDelay_valid_1cycle = OneCycleValid(stageReq.fire, flush) // catch ram data 181 val stageCheck_valid_1cycle = OneCycleValid(stageDelay(1).fire, flush) // replace & perf counter 182 val stageResp_valid_1cycle_dup = Wire(Vec(2, Bool())) 183 stageResp_valid_1cycle_dup.map(_ := OneCycleValid(stageCheck(1).fire, flush)) // ecc flush 184 185 stageReq <> io.req 186 PipelineConnect(stageReq, stageDelay(0), stageDelay(1).ready, flush, rwHarzad) 187 InsideStageConnect(stageDelay(0), stageDelay(1), stageDelay_valid_1cycle) 188 PipelineConnect(stageDelay(1), stageCheck(0), stageCheck(1).ready, flush) 189 InsideStageConnect(stageCheck(0), stageCheck(1), stageCheck_valid_1cycle) 190 PipelineConnect(stageCheck(1), stageResp, io.resp.ready, flush) 191 stageResp.ready := !stageResp.valid || io.resp.ready 192 193 // l3: level 3 non-leaf pte 194 val l3 = if (EnableSv48) Some(Reg(Vec(l2tlbParams.l3Size, new PtwEntry(tagLen = PtwL3TagLen)))) else None 195 val l3v = if (EnableSv48) Some(RegInit(0.U(l2tlbParams.l3Size.W))) else None 196 val l3g = if (EnableSv48) Some(Reg(UInt(l2tlbParams.l3Size.W))) else None 197 val l3asids = if (EnableSv48) Some(l3.get.map(_.asid)) else None 198 val l3vmids = if (EnableSv48) Some(l3.get.map(_.vmid)) else None 199 val l3h = if (EnableSv48) Some(Reg(Vec(l2tlbParams.l3Size, UInt(2.W)))) else None 200 201 // l2: level 2 non-leaf pte 202 val l2 = Reg(Vec(l2tlbParams.l2Size, new PtwEntry(tagLen = PtwL2TagLen))) 203 val l2v = RegInit(0.U(l2tlbParams.l2Size.W)) 204 val l2g = Reg(UInt(l2tlbParams.l2Size.W)) 205 val l2asids = l2.map(_.asid) 206 val l2vmids = l2.map(_.vmid) 207 val l2h = Reg(Vec(l2tlbParams.l2Size, UInt(2.W))) 208 209 // l1: level 1 non-leaf pte 210 val l1 = Module(new SplittedSRAM( 211 l1EntryType, 212 set = l2tlbParams.l1nSets, 213 way = l2tlbParams.l1nWays, 214 waySplit = 2, 215 dataSplit = 4, 216 singlePort = sramSinglePort, 217 readMCP2 = false 218 )) 219 val l1v = RegInit(0.U((l2tlbParams.l1nSets * l2tlbParams.l1nWays).W)) 220 val l1g = Reg(UInt((l2tlbParams.l1nSets * l2tlbParams.l1nWays).W)) 221 val l1h = Reg(Vec(l2tlbParams.l1nSets, Vec(l2tlbParams.l1nWays, UInt(2.W)))) 222 def getl1vSet(vpn: UInt) = { 223 require(log2Up(l2tlbParams.l1nWays) == log2Down(l2tlbParams.l1nWays)) 224 val set = genPtwL1SetIdx(vpn) 225 require(set.getWidth == log2Up(l2tlbParams.l1nSets)) 226 val l1vVec = l1v.asTypeOf(Vec(l2tlbParams.l1nSets, UInt(l2tlbParams.l1nWays.W))) 227 l1vVec(set) 228 } 229 def getl1hSet(vpn: UInt) = { 230 require(log2Up(l2tlbParams.l1nWays) == log2Down(l2tlbParams.l1nWays)) 231 val set = genPtwL1SetIdx(vpn) 232 require(set.getWidth == log2Up(l2tlbParams.l1nSets)) 233 l1h(set) 234 } 235 236 // l0: level 0 leaf pte of 4KB pages 237 val l0 = Module(new SplittedSRAM( 238 l0EntryType, 239 set = l2tlbParams.l0nSets, 240 way = l2tlbParams.l0nWays, 241 waySplit = 4, 242 dataSplit = 4, 243 singlePort = sramSinglePort, 244 readMCP2 = false 245 )) 246 val l0v = RegInit(0.U((l2tlbParams.l0nSets * l2tlbParams.l0nWays).W)) 247 val l0g = Reg(UInt((l2tlbParams.l0nSets * l2tlbParams.l0nWays).W)) 248 val l0h = Reg(Vec(l2tlbParams.l0nSets, Vec(l2tlbParams.l0nWays, UInt(2.W)))) 249 def getl0vSet(vpn: UInt) = { 250 require(log2Up(l2tlbParams.l0nWays) == log2Down(l2tlbParams.l0nWays)) 251 val set = genPtwL0SetIdx(vpn) 252 require(set.getWidth == log2Up(l2tlbParams.l0nSets)) 253 val l0vVec = l0v.asTypeOf(Vec(l2tlbParams.l0nSets, UInt(l2tlbParams.l0nWays.W))) 254 l0vVec(set) 255 } 256 def getl0hSet(vpn: UInt) = { 257 require(log2Up(l2tlbParams.l0nWays) == log2Down(l2tlbParams.l0nWays)) 258 val set = genPtwL0SetIdx(vpn) 259 require(set.getWidth == log2Up(l2tlbParams.l0nSets)) 260 l0h(set) 261 } 262 263 // sp: level 1/2/3 leaf pte of 512GB/1GB/2MB super pages 264 val sp = Reg(Vec(l2tlbParams.spSize, new PtwEntry(tagLen = SPTagLen, hasPerm = true, hasLevel = true))) 265 val spv = RegInit(0.U(l2tlbParams.spSize.W)) 266 val spg = Reg(UInt(l2tlbParams.spSize.W)) 267 val spasids = sp.map(_.asid) 268 val spvmids = sp.map(_.vmid) 269 val sph = Reg(Vec(l2tlbParams.spSize, UInt(2.W))) 270 271 // Access Perf 272 val l3AccessPerf = if(EnableSv48) Some(Wire(Vec(l2tlbParams.l3Size, Bool()))) else None 273 val l2AccessPerf = Wire(Vec(l2tlbParams.l2Size, Bool())) 274 val l1AccessPerf = Wire(Vec(l2tlbParams.l1nWays, Bool())) 275 val l0AccessPerf = Wire(Vec(l2tlbParams.l0nWays, Bool())) 276 val spAccessPerf = Wire(Vec(l2tlbParams.spSize, Bool())) 277 if (EnableSv48) l3AccessPerf.map(_.map(_ := false.B)) 278 l2AccessPerf.map(_ := false.B) 279 l1AccessPerf.map(_ := false.B) 280 l0AccessPerf.map(_ := false.B) 281 spAccessPerf.map(_ := false.B) 282 283 284 285 def vpn_match(vpn1: UInt, vpn2: UInt, level: Int) = { 286 (vpn1(vpnLen-1, vpnnLen*level+3) === vpn2(vpnLen-1, vpnnLen*level+3)) 287 } 288 // NOTE: not actually bypassed, just check if hit, re-access the page cache 289 def refill_bypass(vpn: UInt, level: Int, h_search: UInt) = { 290 val change_h = MuxLookup(h_search, noS2xlate)(Seq( 291 allStage -> onlyStage1, 292 onlyStage1 -> onlyStage1, 293 onlyStage2 -> onlyStage2 294 )) 295 val change_refill_h = MuxLookup(io.refill.bits.req_info_dup(0).s2xlate, noS2xlate)(Seq( 296 allStage -> onlyStage1, 297 onlyStage1 -> onlyStage1, 298 onlyStage2 -> onlyStage2 299 )) 300 val refill_vpn = io.refill.bits.req_info_dup(0).vpn 301 io.refill.valid && (level.U === io.refill.bits.level_dup(0)) && vpn_match(refill_vpn, vpn, level) && change_h === change_refill_h 302 } 303 304 val vpn_search = stageReq.bits.req_info.vpn 305 val h_search = MuxLookup(stageReq.bits.req_info.s2xlate, noS2xlate)(Seq( 306 allStage -> onlyStage1, 307 onlyStage1 -> onlyStage1, 308 onlyStage2 -> onlyStage2 309 )) 310 311 // l3 312 val l3Hit = if(EnableSv48) Some(Wire(Bool())) else None 313 val l3HitPPN = if(EnableSv48) Some(Wire(UInt(ppnLen.W))) else None 314 val l3HitPbmt = if(EnableSv48) Some(Wire(UInt(ptePbmtLen.W))) else None 315 val l3Pre = if(EnableSv48) Some(Wire(Bool())) else None 316 val ptwl3replace = if(EnableSv48) Some(ReplacementPolicy.fromString(l2tlbParams.l3Replacer, l2tlbParams.l3Size)) else None 317 if (EnableSv48) { 318 val hitVecT = l3.get.zipWithIndex.map { 319 case (e, i) => (e.hit(vpn_search, io.csr_dup(2).satp.asid, io.csr_dup(2).vsatp.asid, io.csr_dup(2).hgatp.vmid, s2xlate = h_search =/= noS2xlate) 320 && l3v.get(i) && h_search === l3h.get(i)) 321 } 322 val hitVec = hitVecT.map(RegEnable(_, stageReq.fire)) 323 324 // stageDelay, but check for l3 325 val hitPPN = DataHoldBypass(ParallelPriorityMux(hitVec zip l3.get.map(_.ppn)), stageDelay_valid_1cycle) 326 val hitPbmt = DataHoldBypass(ParallelPriorityMux(hitVec zip l3.get.map(_.pbmt)), stageDelay_valid_1cycle) 327 val hitPre = DataHoldBypass(ParallelPriorityMux(hitVec zip l3.get.map(_.prefetch)), stageDelay_valid_1cycle) 328 val hit = DataHoldBypass(ParallelOR(hitVec), stageDelay_valid_1cycle) 329 330 when (hit && stageDelay_valid_1cycle) { ptwl3replace.get.access(OHToUInt(hitVec)) } 331 332 l3AccessPerf.get.zip(hitVec).map{ case (l, h) => l := h && stageDelay_valid_1cycle} 333 for (i <- 0 until l2tlbParams.l3Size) { 334 XSDebug(stageReq.fire, p"[l3] l3(${i.U}) ${l3.get(i)} hit:${l3.get(i).hit(vpn_search, io.csr_dup(2).satp.asid, io.csr_dup(2).vsatp.asid, io.csr_dup(2).hgatp.vmid, s2xlate = h_search =/= noS2xlate)}\n") 335 } 336 XSDebug(stageReq.fire, p"[l3] l3v:${Binary(l3v.get)} hitVecT:${Binary(VecInit(hitVecT).asUInt)}\n") 337 XSDebug(stageDelay(0).valid, p"[l3] l3Hit:${hit} l3HitPPN:0x${Hexadecimal(hitPPN)} hitVec:${VecInit(hitVec).asUInt}\n") 338 339 VecInit(hitVecT).suggestName(s"l3_hitVecT") 340 VecInit(hitVec).suggestName(s"l3_hitVec") 341 342 // synchronize with other entries with RegEnable 343 l3Hit.map(_ := RegEnable(hit, stageDelay(1).fire)) 344 l3HitPPN.map(_ := RegEnable(hitPPN, stageDelay(1).fire)) 345 l3HitPbmt.map(_ := RegEnable(hitPbmt, stageDelay(1).fire)) 346 l3Pre.map(_ := RegEnable(hitPre, stageDelay(1).fire)) 347 } 348 349 // l2 350 val ptwl2replace = ReplacementPolicy.fromString(l2tlbParams.l2Replacer, l2tlbParams.l2Size) 351 val (l2Hit, l2HitPPN, l2HitPbmt, l2Pre) = { 352 val hitVecT = l2.zipWithIndex.map { 353 case (e, i) => (e.hit(vpn_search, io.csr_dup(2).satp.asid, io.csr_dup(2).vsatp.asid, io.csr_dup(2).hgatp.vmid, s2xlate = h_search =/= noS2xlate) 354 && l2v(i) && h_search === l2h(i)) 355 } 356 val hitVec = hitVecT.map(RegEnable(_, stageReq.fire)) 357 358 // stageDelay, but check for l2 359 val hitPPN = DataHoldBypass(ParallelPriorityMux(hitVec zip l2.map(_.ppn)), stageDelay_valid_1cycle) 360 val hitPbmt = DataHoldBypass(ParallelPriorityMux(hitVec zip l2.map(_.pbmt)), stageDelay_valid_1cycle) 361 val hitPre = DataHoldBypass(ParallelPriorityMux(hitVec zip l2.map(_.prefetch)), stageDelay_valid_1cycle) 362 val hit = DataHoldBypass(ParallelOR(hitVec), stageDelay_valid_1cycle) 363 364 when (hit && stageDelay_valid_1cycle) { ptwl2replace.access(OHToUInt(hitVec)) } 365 366 l2AccessPerf.zip(hitVec).map{ case (l, h) => l := h && stageDelay_valid_1cycle} 367 for (i <- 0 until l2tlbParams.l2Size) { 368 XSDebug(stageReq.fire, p"[l2] l2(${i.U}) ${l2(i)} hit:${l2(i).hit(vpn_search, io.csr_dup(2).satp.asid, io.csr_dup(2).vsatp.asid, io.csr_dup(2).hgatp.vmid, s2xlate = h_search =/= noS2xlate)}\n") 369 } 370 XSDebug(stageReq.fire, p"[l2] l2v:${Binary(l2v)} hitVecT:${Binary(VecInit(hitVecT).asUInt)}\n") 371 XSDebug(stageDelay(0).valid, p"[l2] l2Hit:${hit} l2HitPPN:0x${Hexadecimal(hitPPN)} hitVec:${VecInit(hitVec).asUInt}\n") 372 373 VecInit(hitVecT).suggestName(s"l2_hitVecT") 374 VecInit(hitVec).suggestName(s"l2_hitVec") 375 376 // synchronize with other entries with RegEnable 377 (RegEnable(hit, stageDelay(1).fire), 378 RegEnable(hitPPN, stageDelay(1).fire), 379 RegEnable(hitPbmt, stageDelay(1).fire), 380 RegEnable(hitPre, stageDelay(1).fire)) 381 } 382 383 // l1 384 val ptwl1replace = ReplacementPolicy.fromString(l2tlbParams.l1Replacer,l2tlbParams.l1nWays,l2tlbParams.l1nSets) 385 val (l1Hit, l1HitPPN, l1HitPbmt, l1Pre, l1eccError) = { 386 val ridx = genPtwL1SetIdx(vpn_search) 387 l1.io.r.req.valid := stageReq.fire 388 l1.io.r.req.bits.apply(setIdx = ridx) 389 val vVec_req = getl1vSet(vpn_search) 390 val hVec_req = getl1hSet(vpn_search) 391 392 // delay one cycle after sram read 393 val delay_vpn = stageDelay(0).bits.req_info.vpn 394 val delay_h = MuxLookup(stageDelay(0).bits.req_info.s2xlate, noS2xlate)(Seq( 395 allStage -> onlyStage1, 396 onlyStage1 -> onlyStage1, 397 onlyStage2 -> onlyStage2 398 )) 399 val data_resp = DataHoldBypass(l1.io.r.resp.data, stageDelay_valid_1cycle) 400 val vVec_delay = RegEnable(vVec_req, stageReq.fire) 401 val hVec_delay = RegEnable(hVec_req, stageReq.fire) 402 val hitVec_delay = VecInit(data_resp.zip(vVec_delay.asBools).zip(hVec_delay).map { case ((wayData, v), h) => 403 wayData.entries.hit(delay_vpn, io.csr_dup(1).satp.asid, io.csr_dup(1).vsatp.asid, io.csr_dup(1).hgatp.vmid, s2xlate = delay_h =/= noS2xlate) && v && (delay_h === h)}) 404 405 // check hit and ecc 406 val check_vpn = stageCheck(0).bits.req_info.vpn 407 val ramDatas = RegEnable(data_resp, stageDelay(1).fire) 408 val vVec = RegEnable(vVec_delay, stageDelay(1).fire).asBools 409 410 val hitVec = RegEnable(hitVec_delay, stageDelay(1).fire) 411 val hitWayEntry = ParallelPriorityMux(hitVec zip ramDatas) 412 val hitWayData = hitWayEntry.entries 413 val hit = ParallelOR(hitVec) 414 val hitWay = ParallelPriorityMux(hitVec zip (0 until l2tlbParams.l1nWays).map(_.U(log2Up(l2tlbParams.l1nWays).W))) 415 val eccError = WireInit(false.B) 416 if (l2tlbParams.enablePTWECC) { 417 eccError := hitWayEntry.decode() 418 } else { 419 eccError := false.B 420 } 421 422 ridx.suggestName(s"l1_ridx") 423 ramDatas.suggestName(s"l1_ramDatas") 424 hitVec.suggestName(s"l1_hitVec") 425 hitWayData.suggestName(s"l1_hitWayData") 426 hitWay.suggestName(s"l1_hitWay") 427 428 when (hit && stageCheck_valid_1cycle) { ptwl1replace.access(genPtwL1SetIdx(check_vpn), hitWay) } 429 430 l1AccessPerf.zip(hitVec).map{ case (l, h) => l := h && stageCheck_valid_1cycle } 431 XSDebug(stageDelay_valid_1cycle, p"[l1] ridx:0x${Hexadecimal(ridx)}\n") 432 for (i <- 0 until l2tlbParams.l1nWays) { 433 XSDebug(stageCheck_valid_1cycle, p"[l1] ramDatas(${i.U}) ${ramDatas(i)} l1v:${vVec(i)} hit:${hit}\n") 434 } 435 XSDebug(stageCheck_valid_1cycle, p"[l1] l1Hit:${hit} l1HitPPN:0x${Hexadecimal(hitWayData.ppns(genPtwL1SectorIdx(check_vpn)))} hitVec:${Binary(hitVec.asUInt)} hitWay:${hitWay} vidx:${vVec}\n") 436 437 (hit, hitWayData.ppns(genPtwL1SectorIdx(check_vpn)), hitWayData.pbmts(genPtwL1SectorIdx(check_vpn)), hitWayData.prefetch, eccError) 438 } 439 440 // l0 441 val ptwl0replace = ReplacementPolicy.fromString(l2tlbParams.l0Replacer,l2tlbParams.l0nWays,l2tlbParams.l0nSets) 442 val (l0Hit, l0HitData, l0Pre, l0eccError) = { 443 val ridx = genPtwL0SetIdx(vpn_search) 444 l0.io.r.req.valid := stageReq.fire 445 l0.io.r.req.bits.apply(setIdx = ridx) 446 val vVec_req = getl0vSet(vpn_search) 447 val hVec_req = getl0hSet(vpn_search) 448 449 // delay one cycle after sram read 450 val delay_vpn = stageDelay(0).bits.req_info.vpn 451 val delay_h = MuxLookup(stageDelay(0).bits.req_info.s2xlate, noS2xlate)(Seq( 452 allStage -> onlyStage1, 453 onlyStage1 -> onlyStage1, 454 onlyStage2 -> onlyStage2 455 )) 456 val data_resp = DataHoldBypass(l0.io.r.resp.data, stageDelay_valid_1cycle) 457 val vVec_delay = RegEnable(vVec_req, stageReq.fire) 458 val hVec_delay = RegEnable(hVec_req, stageReq.fire) 459 val hitVec_delay = VecInit(data_resp.zip(vVec_delay.asBools).zip(hVec_delay).map { case ((wayData, v), h) => 460 wayData.entries.hit(delay_vpn, io.csr_dup(0).satp.asid, io.csr_dup(0).vsatp.asid, io.csr_dup(0).hgatp.vmid, s2xlate = delay_h =/= noS2xlate) && v && (delay_h === h)}) 461 462 // check hit and ecc 463 val check_vpn = stageCheck(0).bits.req_info.vpn 464 val ramDatas = RegEnable(data_resp, stageDelay(1).fire) 465 val vVec = RegEnable(vVec_delay, stageDelay(1).fire).asBools 466 467 val hitVec = RegEnable(hitVec_delay, stageDelay(1).fire) 468 val hitWayEntry = ParallelPriorityMux(hitVec zip ramDatas) 469 val hitWayData = hitWayEntry.entries 470 val hitWayEcc = hitWayEntry.ecc 471 val hit = ParallelOR(hitVec) 472 val hitWay = ParallelPriorityMux(hitVec zip (0 until l2tlbParams.l0nWays).map(_.U(log2Up(l2tlbParams.l0nWays).W))) 473 val eccError = WireInit(false.B) 474 if (l2tlbParams.enablePTWECC) { 475 eccError := hitWayEntry.decode() 476 } else { 477 eccError := false.B 478 } 479 480 when (hit && stageCheck_valid_1cycle) { ptwl0replace.access(genPtwL0SetIdx(check_vpn), hitWay) } 481 482 l0AccessPerf.zip(hitVec).map{ case (l, h) => l := h && stageCheck_valid_1cycle } 483 XSDebug(stageReq.fire, p"[l0] ridx:0x${Hexadecimal(ridx)}\n") 484 for (i <- 0 until l2tlbParams.l0nWays) { 485 XSDebug(stageCheck_valid_1cycle, p"[l0] ramDatas(${i.U}) ${ramDatas(i)} l0v:${vVec(i)} hit:${hitVec(i)}\n") 486 } 487 XSDebug(stageCheck_valid_1cycle, p"[l0] l0Hit:${hit} l0HitData:${hitWayData} hitVec:${Binary(hitVec.asUInt)} hitWay:${hitWay} v:${vVec}\n") 488 489 ridx.suggestName(s"l0_ridx") 490 ramDatas.suggestName(s"l0_ramDatas") 491 hitVec.suggestName(s"l0_hitVec") 492 hitWay.suggestName(s"l0_hitWay") 493 494 (hit, hitWayData, hitWayData.prefetch, eccError) 495 } 496 val l0HitPPN = l0HitData.ppns 497 val l0HitPbmt = l0HitData.pbmts 498 val l0HitPerm = l0HitData.perms.getOrElse(0.U.asTypeOf(Vec(PtwL0SectorSize, new PtePermBundle))) 499 val l0HitValid = VecInit(l0HitData.onlypf.map(!_)) 500 501 // super page 502 val spreplace = ReplacementPolicy.fromString(l2tlbParams.spReplacer, l2tlbParams.spSize) 503 val (spHit, spHitData, spPre, spValid) = { 504 val hitVecT = sp.zipWithIndex.map { case (e, i) => e.hit(vpn_search, io.csr_dup(0).satp.asid, io.csr_dup(0).vsatp.asid, io.csr_dup(0).hgatp.vmid, s2xlate = h_search =/= noS2xlate) && spv(i) && (sph(i) === h_search) } 505 val hitVec = hitVecT.map(RegEnable(_, stageReq.fire)) 506 val hitData = ParallelPriorityMux(hitVec zip sp) 507 val hit = ParallelOR(hitVec) 508 509 when (hit && stageDelay_valid_1cycle) { spreplace.access(OHToUInt(hitVec)) } 510 511 spAccessPerf.zip(hitVec).map{ case (s, h) => s := h && stageDelay_valid_1cycle } 512 for (i <- 0 until l2tlbParams.spSize) { 513 XSDebug(stageReq.fire, p"[sp] sp(${i.U}) ${sp(i)} hit:${sp(i).hit(vpn_search, io.csr_dup(0).satp.asid, io.csr_dup(0).vsatp.asid, io.csr_dup(0).hgatp.vmid, s2xlate = h_search =/= noS2xlate)} spv:${spv(i)}\n") 514 } 515 XSDebug(stageDelay_valid_1cycle, p"[sp] spHit:${hit} spHitData:${hitData} hitVec:${Binary(VecInit(hitVec).asUInt)}\n") 516 517 VecInit(hitVecT).suggestName(s"sp_hitVecT") 518 VecInit(hitVec).suggestName(s"sp_hitVec") 519 520 (RegEnable(hit, stageDelay(1).fire), 521 RegEnable(hitData, stageDelay(1).fire), 522 RegEnable(hitData.prefetch, stageDelay(1).fire), 523 RegEnable(hitData.v, stageDelay(1).fire)) 524 } 525 val spHitPerm = spHitData.perm.getOrElse(0.U.asTypeOf(new PtePermBundle)) 526 val spHitLevel = spHitData.level.getOrElse(0.U) 527 528 val check_res = Wire(new PageCacheRespBundle) 529 check_res.l3.map(_.apply(l3Hit.get, l3Pre.get, l3HitPPN.get)) 530 check_res.l2.apply(l2Hit, l2Pre, l2HitPPN, l2HitPbmt) 531 check_res.l1.apply(l1Hit, l1Pre, l1HitPPN, l1HitPbmt, ecc = l1eccError) 532 check_res.l0.apply(l0Hit, l0Pre, l0HitPPN, l0HitPbmt, l0HitPerm, l0eccError, valid = l0HitValid) 533 check_res.sp.apply(spHit, spPre, spHitData.ppn, spHitData.pbmt, spHitPerm, false.B, spHitLevel, spValid) 534 535 val resp_res = Reg(new PageCacheRespBundle) 536 when (stageCheck(1).fire) { resp_res := check_res } 537 538 // stageResp bypass 539 val bypassed = if (EnableSv48) Wire(Vec(4, Bool())) else Wire(Vec(3, Bool())) 540 bypassed.indices.foreach(i => 541 bypassed(i) := stageResp.bits.bypassed(i) || 542 ValidHoldBypass(refill_bypass(stageResp.bits.req_info.vpn, i, stageResp.bits.req_info.s2xlate), 543 OneCycleValid(stageCheck(1).fire, false.B) || io.refill.valid) 544 ) 545 546 // stageResp bypass to hptw 547 val hptw_bypassed = if (EnableSv48) Wire(Vec(4, Bool())) else Wire(Vec(3, Bool())) 548 hptw_bypassed.indices.foreach(i => 549 hptw_bypassed(i) := stageResp.bits.bypassed(i) || 550 ValidHoldBypass(refill_bypass(stageResp.bits.req_info.vpn, i, stageResp.bits.req_info.s2xlate), 551 io.resp.fire) 552 ) 553 554 val isAllStage = stageResp.bits.req_info.s2xlate === allStage 555 val isOnlyStage2 = stageResp.bits.req_info.s2xlate === onlyStage2 556 val stage1Hit = (resp_res.l0.hit || resp_res.sp.hit) && isAllStage 557 val idx = stageResp.bits.req_info.vpn(2, 0) 558 val stage1Pf = !Mux(resp_res.l0.hit, resp_res.l0.v(idx), resp_res.sp.v) 559 io.resp.bits.req_info := stageResp.bits.req_info 560 io.resp.bits.isFirst := stageResp.bits.isFirst 561 io.resp.bits.hit := (resp_res.l0.hit || resp_res.sp.hit) && (!isAllStage || isAllStage && stage1Pf) 562 if (EnableSv48) { 563 io.resp.bits.bypassed := ((bypassed(0) && !resp_res.l0.hit) || (bypassed(1) && !resp_res.l1.hit) || (bypassed(2) && !resp_res.l2.hit) || (bypassed(3) && !resp_res.l3.get.hit)) && !isAllStage 564 } else { 565 io.resp.bits.bypassed := ((bypassed(0) && !resp_res.l0.hit) || (bypassed(1) && !resp_res.l1.hit) || (bypassed(2) && !resp_res.l2.hit)) && !isAllStage 566 } 567 io.resp.bits.prefetch := resp_res.l0.pre && resp_res.l0.hit || resp_res.sp.pre && resp_res.sp.hit 568 io.resp.bits.toFsm.l3Hit.map(_ := resp_res.l3.get.hit && !stage1Hit && !isOnlyStage2 && !stageResp.bits.isHptwReq) 569 io.resp.bits.toFsm.l2Hit := resp_res.l2.hit && !stage1Hit && !isOnlyStage2 && !stageResp.bits.isHptwReq 570 io.resp.bits.toFsm.l1Hit := resp_res.l1.hit && !stage1Hit && !isOnlyStage2 && !stageResp.bits.isHptwReq 571 io.resp.bits.toFsm.ppn := Mux(resp_res.l1.hit, resp_res.l1.ppn, Mux(resp_res.l2.hit, resp_res.l2.ppn, resp_res.l3.getOrElse(0.U.asTypeOf(new PageCachePerPespBundle)).ppn)) 572 io.resp.bits.toFsm.stage1Hit := stage1Hit 573 574 io.resp.bits.isHptwReq := stageResp.bits.isHptwReq 575 if (EnableSv48) { 576 io.resp.bits.toHptw.bypassed := ((hptw_bypassed(0) && !resp_res.l0.hit) || (hptw_bypassed(1) && !resp_res.l1.hit) || (hptw_bypassed(2) && !resp_res.l2.hit) || (hptw_bypassed(3) && !resp_res.l3.get.hit)) && stageResp.bits.isHptwReq 577 } else { 578 io.resp.bits.toHptw.bypassed := ((hptw_bypassed(0) && !resp_res.l0.hit) || (hptw_bypassed(1) && !resp_res.l1.hit) || (hptw_bypassed(2) && !resp_res.l2.hit)) && stageResp.bits.isHptwReq 579 } 580 io.resp.bits.toHptw.id := stageResp.bits.hptwId 581 io.resp.bits.toHptw.l3Hit.map(_ := resp_res.l3.get.hit && stageResp.bits.isHptwReq) 582 io.resp.bits.toHptw.l2Hit := resp_res.l2.hit && stageResp.bits.isHptwReq 583 io.resp.bits.toHptw.l1Hit := resp_res.l1.hit && stageResp.bits.isHptwReq 584 io.resp.bits.toHptw.ppn := Mux(resp_res.l1.hit, resp_res.l1.ppn, Mux(resp_res.l2.hit, resp_res.l2.ppn, resp_res.l3.getOrElse(0.U.asTypeOf(new PageCachePerPespBundle)).ppn))(ppnLen - 1, 0) 585 io.resp.bits.toHptw.resp.entry.tag := stageResp.bits.req_info.vpn 586 io.resp.bits.toHptw.resp.entry.asid := DontCare 587 io.resp.bits.toHptw.resp.entry.vmid.map(_ := io.csr_dup(0).hgatp.vmid) 588 io.resp.bits.toHptw.resp.entry.level.map(_ := Mux(resp_res.l0.hit, 0.U, resp_res.sp.level)) 589 io.resp.bits.toHptw.resp.entry.prefetch := from_pre(stageResp.bits.req_info.source) 590 io.resp.bits.toHptw.resp.entry.ppn := Mux(resp_res.l0.hit, resp_res.l0.ppn(idx), resp_res.sp.ppn)(ppnLen - 1, 0) 591 io.resp.bits.toHptw.resp.entry.pbmt := Mux(resp_res.l0.hit, resp_res.l0.pbmt(idx), resp_res.sp.pbmt) 592 io.resp.bits.toHptw.resp.entry.perm.map(_ := Mux(resp_res.l0.hit, resp_res.l0.perm(idx), resp_res.sp.perm)) 593 io.resp.bits.toHptw.resp.entry.v := Mux(resp_res.l0.hit, resp_res.l0.v(idx), resp_res.sp.v) 594 io.resp.bits.toHptw.resp.gpf := !io.resp.bits.toHptw.resp.entry.v 595 io.resp.bits.toHptw.resp.gaf := false.B 596 597 io.resp.bits.stage1.entry.map(_.tag := stageResp.bits.req_info.vpn(vpnLen - 1, 3)) 598 io.resp.bits.stage1.entry.map(_.asid := Mux(stageResp.bits.req_info.hasS2xlate(), io.csr_dup(0).vsatp.asid, io.csr_dup(0).satp.asid)) // DontCare 599 io.resp.bits.stage1.entry.map(_.vmid.map(_ := io.csr_dup(0).hgatp.vmid)) 600 if (EnableSv48) { 601 io.resp.bits.stage1.entry.map(_.level.map(_ := Mux(resp_res.l0.hit, 0.U, 602 Mux(resp_res.sp.hit, resp_res.sp.level, 603 Mux(resp_res.l1.hit, 1.U, 604 Mux(resp_res.l2.hit, 2.U, 3.U)))))) 605 } else { 606 io.resp.bits.stage1.entry.map(_.level.map(_ := Mux(resp_res.l0.hit, 0.U, 607 Mux(resp_res.sp.hit, resp_res.sp.level, 608 Mux(resp_res.l1.hit, 1.U, 2.U))))) 609 } 610 io.resp.bits.stage1.entry.map(_.prefetch := from_pre(stageResp.bits.req_info.source)) 611 for (i <- 0 until tlbcontiguous) { 612 if (EnableSv48) { 613 io.resp.bits.stage1.entry(i).ppn := Mux(resp_res.l0.hit, resp_res.l0.ppn(i)(gvpnLen - 1, sectortlbwidth), 614 Mux(resp_res.sp.hit, resp_res.sp.ppn(gvpnLen - 1, sectortlbwidth), 615 Mux(resp_res.l1.hit, resp_res.l1.ppn(gvpnLen - 1, sectortlbwidth), 616 Mux(resp_res.l2.hit, resp_res.l2.ppn(gvpnLen - 1, sectortlbwidth), 617 resp_res.l3.get.ppn(gvpnLen - 1, sectortlbwidth))))) 618 io.resp.bits.stage1.entry(i).ppn_low := Mux(resp_res.l0.hit, resp_res.l0.ppn(i)(sectortlbwidth - 1, 0), 619 Mux(resp_res.sp.hit, resp_res.sp.ppn(sectortlbwidth - 1, 0), 620 Mux(resp_res.l1.hit, resp_res.l1.ppn(sectortlbwidth - 1, 0), 621 Mux(resp_res.l2.hit, resp_res.l2.ppn(sectortlbwidth - 1, 0), 622 resp_res.l3.get.ppn(sectortlbwidth - 1, 0))))) 623 io.resp.bits.stage1.entry(i).v := Mux(resp_res.l0.hit, resp_res.l0.v(i), 624 Mux(resp_res.sp.hit, resp_res.sp.v, 625 Mux(resp_res.l1.hit, resp_res.l1.v, 626 Mux(resp_res.l2.hit, resp_res.l2.v, 627 resp_res.l3.get.v)))) 628 } else { 629 io.resp.bits.stage1.entry(i).ppn := Mux(resp_res.l0.hit, resp_res.l0.ppn(i)(gvpnLen - 1, sectortlbwidth), 630 Mux(resp_res.sp.hit, resp_res.sp.ppn(gvpnLen - 1, sectortlbwidth), 631 Mux(resp_res.l1.hit, resp_res.l1.ppn(gvpnLen - 1, sectortlbwidth), 632 resp_res.l2.ppn(gvpnLen - 1, sectortlbwidth)))) 633 io.resp.bits.stage1.entry(i).ppn_low := Mux(resp_res.l0.hit, resp_res.l0.ppn(i)(sectortlbwidth - 1, 0), 634 Mux(resp_res.sp.hit, resp_res.sp.ppn(sectortlbwidth - 1, 0), 635 Mux(resp_res.l1.hit, resp_res.l1.ppn(sectortlbwidth - 1, 0), 636 resp_res.l2.ppn(sectortlbwidth - 1, 0)))) 637 io.resp.bits.stage1.entry(i).v := Mux(resp_res.l0.hit, resp_res.l0.v(i), 638 Mux(resp_res.sp.hit, resp_res.sp.v, 639 Mux(resp_res.l1.hit, resp_res.l1.v, 640 resp_res.l2.v))) 641 } 642 io.resp.bits.stage1.entry(i).pbmt := Mux(resp_res.l0.hit, resp_res.l0.pbmt(i), 643 Mux(resp_res.sp.hit, resp_res.sp.pbmt, 644 Mux(resp_res.l1.hit, resp_res.l1.pbmt, 645 resp_res.l2.pbmt))) 646 io.resp.bits.stage1.entry(i).perm.map(_ := Mux(resp_res.l0.hit, resp_res.l0.perm(i), Mux(resp_res.sp.hit, resp_res.sp.perm, 0.U.asTypeOf(new PtePermBundle)))) 647 io.resp.bits.stage1.entry(i).pf := !io.resp.bits.stage1.entry(i).v 648 io.resp.bits.stage1.entry(i).af := false.B 649 } 650 io.resp.bits.stage1.pteidx := UIntToOH(idx).asBools 651 io.resp.bits.stage1.not_super := Mux(resp_res.l0.hit, true.B, false.B) 652 io.resp.bits.stage1.not_merge := false.B 653 io.resp.valid := stageResp.valid 654 XSError(stageResp.valid && resp_res.l0.hit && resp_res.sp.hit, "normal page and super page both hit") 655 656 // refill Perf 657 val l3RefillPerf = if (EnableSv48) Some(Wire(Vec(l2tlbParams.l3Size, Bool()))) else None 658 val l2RefillPerf = Wire(Vec(l2tlbParams.l2Size, Bool())) 659 val l1RefillPerf = Wire(Vec(l2tlbParams.l1nWays, Bool())) 660 val l0RefillPerf = Wire(Vec(l2tlbParams.l0nWays, Bool())) 661 val spRefillPerf = Wire(Vec(l2tlbParams.spSize, Bool())) 662 l3RefillPerf.map(_.map(_ := false.B)) 663 l2RefillPerf.map(_ := false.B) 664 l1RefillPerf.map(_ := false.B) 665 l0RefillPerf.map(_ := false.B) 666 spRefillPerf.map(_ := false.B) 667 668 // refill 669 l1.io.w.req <> DontCare 670 l0.io.w.req <> DontCare 671 l1.io.w.req.valid := false.B 672 l0.io.w.req.valid := false.B 673 674 val memRdata = refill.ptes 675 val memPtes = (0 until (l2tlbParams.blockBytes/(XLEN/8))).map(i => memRdata((i+1)*XLEN-1, i*XLEN).asTypeOf(new PteBundle)) 676 val memSelData = io.refill.bits.sel_pte_dup 677 val memPte = memSelData.map(a => a.asTypeOf(new PteBundle)) 678 val mPBMTE = io.csr.mPBMTE 679 val hPBMTE = io.csr.hPBMTE 680 val pbmte = Mux(refill.req_info_dup(0).s2xlate === onlyStage1 || refill.req_info_dup(0).s2xlate === allStage, hPBMTE, mPBMTE) 681 682 // TODO: handle sfenceLatch outsize 683 if (EnableSv48) { 684 when ( 685 !flush_dup(2) && 686 refill.levelOH.l3.get && 687 !memPte(2).isLeaf() && 688 memPte(2).canRefill(refill.level_dup(2), refill.req_info_dup(2).s2xlate, pbmte, io.csr_dup(2).vsatp.mode) 689 ) { 690 val refillIdx = replaceWrapper(l3v.get, ptwl3replace.get.way) 691 refillIdx.suggestName(s"Ptwl3RefillIdx") 692 val rfOH = UIntToOH(refillIdx) 693 l3.get(refillIdx).refill( 694 refill.req_info_dup(2).vpn, 695 Mux(refill.req_info_dup(2).s2xlate =/= noS2xlate, io.csr_dup(2).vsatp.asid, io.csr_dup(2).satp.asid), 696 io.csr_dup(2).hgatp.vmid, 697 memSelData(2), 698 3.U, 699 refill_prefetch_dup(2) 700 ) 701 ptwl2replace.access(refillIdx) 702 l3v.get := l3v.get | rfOH 703 l3g.get := (l3g.get & ~rfOH) | Mux(memPte(2).perm.g, rfOH, 0.U) 704 l3h.get(refillIdx) := refill_h(2) 705 706 for (i <- 0 until l2tlbParams.l3Size) { 707 l3RefillPerf.get(i) := i.U === refillIdx 708 } 709 710 XSDebug(p"[l3 refill] refillIdx:${refillIdx} refillEntry:${l3.get(refillIdx).genPtwEntry(refill.req_info_dup(2).vpn, Mux(refill.req_info_dup(2).s2xlate =/= noS2xlate, io.csr_dup(2).vsatp.asid, io.csr_dup(2).satp.asid), memSelData(2), 0.U, prefetch = refill_prefetch_dup(2))}\n") 711 XSDebug(p"[l3 refill] l3v:${Binary(l3v.get)}->${Binary(l3v.get | rfOH)} l3g:${Binary(l3g.get)}->${Binary((l3g.get & ~rfOH) | Mux(memPte(2).perm.g, rfOH, 0.U))}\n") 712 713 refillIdx.suggestName(s"l3_refillIdx") 714 rfOH.suggestName(s"l3_rfOH") 715 } 716 } 717 718 // L2 refill 719 when ( 720 !flush_dup(2) && 721 refill.levelOH.l2 && 722 !memPte(2).isLeaf() && 723 memPte(2).canRefill(refill.level_dup(2), refill.req_info_dup(2).s2xlate, pbmte, io.csr_dup(2).vsatp.mode) 724 ) { 725 val refillIdx = replaceWrapper(l2v, ptwl2replace.way) 726 refillIdx.suggestName(s"Ptwl2RefillIdx") 727 val rfOH = UIntToOH(refillIdx) 728 l2(refillIdx).refill( 729 refill.req_info_dup(2).vpn, 730 Mux(refill.req_info_dup(2).s2xlate =/= noS2xlate, io.csr_dup(2).vsatp.asid, io.csr_dup(2).satp.asid), 731 io.csr_dup(2).hgatp.vmid, 732 memSelData(2), 733 2.U, 734 refill_prefetch_dup(2) 735 ) 736 ptwl2replace.access(refillIdx) 737 l2v := l2v | rfOH 738 l2g := (l2g & ~rfOH) | Mux(memPte(2).perm.g, rfOH, 0.U) 739 l2h(refillIdx) := refill_h(2) 740 741 for (i <- 0 until l2tlbParams.l2Size) { 742 l2RefillPerf(i) := i.U === refillIdx 743 } 744 745 XSDebug(p"[l2 refill] refillIdx:${refillIdx} refillEntry:${l2(refillIdx).genPtwEntry(refill.req_info_dup(2).vpn, Mux(refill.req_info_dup(2).s2xlate =/= noS2xlate, io.csr_dup(2).vsatp.asid, io.csr_dup(2).satp.asid), memSelData(2), 0.U, prefetch = refill_prefetch_dup(2))}\n") 746 XSDebug(p"[l2 refill] l2v:${Binary(l2v)}->${Binary(l2v | rfOH)} l2g:${Binary(l2g)}->${Binary((l2g & ~rfOH) | Mux(memPte(2).perm.g, rfOH, 0.U))}\n") 747 748 refillIdx.suggestName(s"l2_refillIdx") 749 rfOH.suggestName(s"l2_rfOH") 750 } 751 752 // L1 refill 753 when (!flush_dup(1) && refill.levelOH.l1) { 754 val refillIdx = genPtwL1SetIdx(refill.req_info_dup(1).vpn) 755 val victimWay = replaceWrapper(getl1vSet(refill.req_info_dup(1).vpn), ptwl1replace.way(refillIdx)) 756 val victimWayOH = UIntToOH(victimWay) 757 val rfvOH = UIntToOH(Cat(refillIdx, victimWay)) 758 val wdata = Wire(l1EntryType) 759 wdata.gen( 760 vpn = refill.req_info_dup(1).vpn, 761 asid = Mux(refill.req_info_dup(1).s2xlate =/= noS2xlate, io.csr_dup(1).vsatp.asid, io.csr_dup(1).satp.asid), 762 vmid = io.csr_dup(1).hgatp.vmid, 763 data = memRdata, 764 levelUInt = 1.U, 765 refill_prefetch_dup(1), 766 refill.req_info_dup(1).s2xlate, 767 pbmte, 768 io.csr_dup(1).vsatp.mode 769 ) 770 l1.io.w.apply( 771 valid = true.B, 772 setIdx = refillIdx, 773 data = wdata, 774 waymask = victimWayOH 775 ) 776 ptwl1replace.access(refillIdx, victimWay) 777 l1v := l1v | rfvOH 778 l1g := l1g & ~rfvOH | Mux(Cat(memPtes.map(_.perm.g)).andR, rfvOH, 0.U) 779 l1h(refillIdx)(victimWay) := refill_h(1) 780 781 for (i <- 0 until l2tlbParams.l1nWays) { 782 l1RefillPerf(i) := i.U === victimWay 783 } 784 785 XSDebug(p"[l1 refill] refillIdx:0x${Hexadecimal(refillIdx)} victimWay:${victimWay} victimWayOH:${Binary(victimWayOH)} rfvOH(in UInt):${Cat(refillIdx, victimWay)}\n") 786 XSDebug(p"[l1 refill] refilldata:0x${wdata}\n") 787 XSDebug(p"[l1 refill] l1v:${Binary(l1v)} -> ${Binary(l1v | rfvOH)}\n") 788 XSDebug(p"[l1 refill] l1g:${Binary(l1g)} -> ${Binary(l1g & ~rfvOH | Mux(Cat(memPtes.map(_.perm.g)).andR, rfvOH, 0.U))}\n") 789 790 refillIdx.suggestName(s"l1_refillIdx") 791 victimWay.suggestName(s"l1_victimWay") 792 victimWayOH.suggestName(s"l1_victimWayOH") 793 rfvOH.suggestName(s"l1_rfvOH") 794 } 795 796 // L0 refill 797 when (!flush_dup(0) && refill.levelOH.l0) { 798 val refillIdx = genPtwL0SetIdx(refill.req_info_dup(0).vpn) 799 val victimWay = replaceWrapper(getl0vSet(refill.req_info_dup(0).vpn), ptwl0replace.way(refillIdx)) 800 val victimWayOH = UIntToOH(victimWay) 801 val rfvOH = UIntToOH(Cat(refillIdx, victimWay)) 802 val wdata = Wire(l0EntryType) 803 wdata.gen( 804 vpn = refill.req_info_dup(0).vpn, 805 asid = Mux(refill.req_info_dup(0).s2xlate =/= noS2xlate, io.csr_dup(0).vsatp.asid, io.csr_dup(0).satp.asid), 806 vmid = io.csr_dup(0).hgatp.vmid, 807 data = memRdata, 808 levelUInt = 0.U, 809 refill_prefetch_dup(0), 810 refill.req_info_dup(0).s2xlate, 811 pbmte, 812 io.csr_dup(0).vsatp.mode 813 ) 814 l0.io.w.apply( 815 valid = true.B, 816 setIdx = refillIdx, 817 data = wdata, 818 waymask = victimWayOH 819 ) 820 ptwl0replace.access(refillIdx, victimWay) 821 l0v := l0v | rfvOH 822 l0g := l0g & ~rfvOH | Mux(Cat(memPtes.map(_.perm.g)).andR, rfvOH, 0.U) 823 l0h(refillIdx)(victimWay) := refill_h(0) 824 825 for (i <- 0 until l2tlbParams.l0nWays) { 826 l0RefillPerf(i) := i.U === victimWay 827 } 828 829 XSDebug(p"[l0 refill] refillIdx:0x${Hexadecimal(refillIdx)} victimWay:${victimWay} victimWayOH:${Binary(victimWayOH)} rfvOH(in UInt):${Cat(refillIdx, victimWay)}\n") 830 XSDebug(p"[l0 refill] refilldata:0x${wdata}\n") 831 XSDebug(p"[l0 refill] l0v:${Binary(l0v)} -> ${Binary(l0v | rfvOH)}\n") 832 XSDebug(p"[l0 refill] l0g:${Binary(l0g)} -> ${Binary(l0g & ~rfvOH | Mux(Cat(memPtes.map(_.perm.g)).andR, rfvOH, 0.U))}\n") 833 834 refillIdx.suggestName(s"l0_refillIdx") 835 victimWay.suggestName(s"l0_victimWay") 836 victimWayOH.suggestName(s"l0_victimWayOH") 837 rfvOH.suggestName(s"l0_rfvOH") 838 } 839 840 841 // misc entries: super & invalid 842 when ( 843 !flush_dup(0) && 844 refill.levelOH.sp && 845 ((memPte(0).isLeaf() && memPte(0).canRefill(refill.level_dup(0), refill.req_info_dup(0).s2xlate, pbmte, io.csr_dup(0).vsatp.mode)) || 846 memPte(0).onlyPf(refill.level_dup(0), refill.req_info_dup(0).s2xlate, pbmte)) 847 ) { 848 val refillIdx = spreplace.way// LFSR64()(log2Up(l2tlbParams.spSize)-1,0) // TODO: may be LRU 849 val rfOH = UIntToOH(refillIdx) 850 sp(refillIdx).refill( 851 refill.req_info_dup(0).vpn, 852 Mux(refill.req_info_dup(0).s2xlate =/= noS2xlate, io.csr_dup(0).vsatp.asid, io.csr_dup(0).satp.asid), 853 io.csr_dup(0).hgatp.vmid, 854 memSelData(0), 855 refill.level_dup(0), 856 refill_prefetch_dup(0), 857 !memPte(0).onlyPf(refill.level_dup(0), refill.req_info_dup(0).s2xlate, pbmte) 858 ) 859 spreplace.access(refillIdx) 860 spv := spv | rfOH 861 spg := spg & ~rfOH | Mux(memPte(0).perm.g, rfOH, 0.U) 862 sph(refillIdx) := refill_h(0) 863 864 for (i <- 0 until l2tlbParams.spSize) { 865 spRefillPerf(i) := i.U === refillIdx 866 } 867 868 XSDebug(p"[sp refill] refillIdx:${refillIdx} refillEntry:${sp(refillIdx).genPtwEntry(refill.req_info_dup(0).vpn, Mux(refill.req_info_dup(0).s2xlate =/= noS2xlate, io.csr_dup(0).vsatp.asid, io.csr_dup(0).satp.asid), memSelData(0), refill.level_dup(0), refill_prefetch_dup(0))}\n") 869 XSDebug(p"[sp refill] spv:${Binary(spv)}->${Binary(spv | rfOH)} spg:${Binary(spg)}->${Binary(spg & ~rfOH | Mux(memPte(0).perm.g, rfOH, 0.U))}\n") 870 871 refillIdx.suggestName(s"sp_refillIdx") 872 rfOH.suggestName(s"sp_rfOH") 873 } 874 875 val l1eccFlush = resp_res.l1.ecc && stageResp_valid_1cycle_dup(0) // RegNext(l1eccError, init = false.B) 876 val l0eccFlush = resp_res.l0.ecc && stageResp_valid_1cycle_dup(1) // RegNext(l0eccError, init = false.B) 877 val eccVpn = stageResp.bits.req_info.vpn 878 879 XSError(l1eccFlush, "l2tlb.cache.l1 ecc error. Should not happen at sim stage") 880 XSError(l0eccFlush, "l2tlb.cache.l0 ecc error. Should not happen at sim stage") 881 when (l1eccFlush) { 882 val flushSetIdxOH = UIntToOH(genPtwL1SetIdx(eccVpn)) 883 val flushMask = VecInit(flushSetIdxOH.asBools.map { a => Fill(l2tlbParams.l1nWays, a.asUInt) }).asUInt 884 l1v := l1v & ~flushMask 885 l1g := l1g & ~flushMask 886 } 887 888 when (l0eccFlush) { 889 val flushSetIdxOH = UIntToOH(genPtwL0SetIdx(eccVpn)) 890 val flushMask = VecInit(flushSetIdxOH.asBools.map { a => Fill(l2tlbParams.l0nWays, a.asUInt) }).asUInt 891 l0v := l0v & ~flushMask 892 l0g := l0g & ~flushMask 893 } 894 895 // sfence for l0 896 val sfence_valid_l0 = sfence_dup(0).valid && !sfence_dup(0).bits.hg && !sfence_dup(0).bits.hv 897 when (sfence_valid_l0) { 898 val l0hhit = VecInit(l0h.flatMap(_.map{a => io.csr_dup(0).priv.virt && a === onlyStage1 || !io.csr_dup(0).priv.virt && a === noS2xlate})).asUInt 899 val sfence_vpn = sfence_dup(0).bits.addr(sfence_dup(0).bits.addr.getWidth-1, offLen) 900 when (sfence_dup(0).bits.rs1/*va*/) { 901 when (sfence_dup(0).bits.rs2) { 902 // all va && all asid 903 l0v := l0v & ~l0hhit 904 } .otherwise { 905 // all va && specific asid except global 906 l0v := l0v & (l0g | ~l0hhit) 907 } 908 } .otherwise { 909 // val flushMask = UIntToOH(genTlbl1Idx(sfence.bits.addr(sfence.bits.addr.getWidth-1, offLen))) 910 val flushSetIdxOH = UIntToOH(genPtwL0SetIdx(sfence_vpn)) 911 // val flushMask = VecInit(flushSetIdxOH.asBools.map(Fill(l2tlbParams.l0nWays, _.asUInt))).asUInt 912 val flushMask = VecInit(flushSetIdxOH.asBools.map { a => Fill(l2tlbParams.l0nWays, a.asUInt) }).asUInt 913 flushSetIdxOH.suggestName(s"sfence_nrs1_flushSetIdxOH") 914 flushMask.suggestName(s"sfence_nrs1_flushMask") 915 916 when (sfence_dup(0).bits.rs2) { 917 // specific leaf of addr && all asid 918 l0v := l0v & ~flushMask & ~l0hhit 919 } .otherwise { 920 // specific leaf of addr && specific asid 921 l0v := l0v & (~flushMask | l0g | ~l0hhit) 922 } 923 } 924 } 925 926 // hfencev, simple implementation for l0 927 val hfencev_valid_l0 = sfence_dup(0).valid && sfence_dup(0).bits.hv 928 when(hfencev_valid_l0) { 929 val flushMask = VecInit(l0h.flatMap(_.map(_ === onlyStage1))).asUInt 930 l0v := l0v & ~flushMask // all VS-stage l0 pte 931 } 932 933 // hfenceg, simple implementation for l0 934 val hfenceg_valid_l0 = sfence_dup(0).valid && sfence_dup(0).bits.hg 935 when(hfenceg_valid_l0) { 936 val flushMask = VecInit(l0h.flatMap(_.map(_ === onlyStage2))).asUInt 937 l0v := l0v & ~flushMask // all G-stage l0 pte 938 } 939 940 val l2asidhit = VecInit(l2asids.map(_ === sfence_dup(2).bits.id)).asUInt 941 val spasidhit = VecInit(spasids.map(_ === sfence_dup(0).bits.id)).asUInt 942 val sfence_valid = sfence_dup(0).valid && !sfence_dup(0).bits.hg && !sfence_dup(0).bits.hv 943 when (sfence_valid) { 944 val l2vmidhit = VecInit(l2vmids.map(_.getOrElse(0.U) === io.csr_dup(2).hgatp.vmid)).asUInt 945 val spvmidhit = VecInit(spvmids.map(_.getOrElse(0.U) === io.csr_dup(0).hgatp.vmid)).asUInt 946 val l2hhit = VecInit(l2h.map{a => io.csr_dup(2).priv.virt && a === onlyStage1 || !io.csr_dup(2).priv.virt && a === noS2xlate}).asUInt 947 val sphhit = VecInit(sph.map{a => io.csr_dup(0).priv.virt && a === onlyStage1 || !io.csr_dup(0).priv.virt && a === noS2xlate}).asUInt 948 val l1hhit = VecInit(l1h.flatMap(_.map{a => io.csr_dup(1).priv.virt && a === onlyStage1 || !io.csr_dup(1).priv.virt && a === noS2xlate})).asUInt 949 val sfence_vpn = sfence_dup(0).bits.addr(sfence_dup(0).bits.addr.getWidth-1, offLen) 950 951 when (sfence_dup(0).bits.rs1/*va*/) { 952 when (sfence_dup(0).bits.rs2) { 953 // all va && all asid 954 l1v := l1v & ~l1hhit 955 l2v := l2v & ~(l2hhit & VecInit(l2vmidhit.asBools.map{a => io.csr_dup(2).priv.virt && a || !io.csr_dup(2).priv.virt}).asUInt) 956 spv := spv & ~(sphhit & VecInit(spvmidhit.asBools.map{a => io.csr_dup(0).priv.virt && a || !io.csr_dup(0).priv.virt}).asUInt) 957 } .otherwise { 958 // all va && specific asid except global 959 l1v := l1v & (l1g | ~l1hhit) 960 l2v := l2v & ~(~l2g & l2hhit & l2asidhit & VecInit(l2vmidhit.asBools.map{a => io.csr_dup(2).priv.virt && a || !io.csr_dup(2).priv.virt}).asUInt) 961 spv := spv & ~(~spg & sphhit & spasidhit & VecInit(spvmidhit.asBools.map{a => io.csr_dup(0).priv.virt && a || !io.csr_dup(0).priv.virt}).asUInt) 962 } 963 } .otherwise { 964 when (sfence_dup(0).bits.rs2) { 965 // specific leaf of addr && all asid 966 spv := spv & ~(sphhit & VecInit(sp.map(_.hit(sfence_vpn, sfence_dup(0).bits.id, sfence_dup(0).bits.id, io.csr_dup(0).hgatp.vmid, ignoreAsid = true, s2xlate = io.csr_dup(0).priv.virt))).asUInt) 967 } .otherwise { 968 // specific leaf of addr && specific asid 969 spv := spv & ~(~spg & sphhit & VecInit(sp.map(_.hit(sfence_vpn, sfence_dup(0).bits.id, sfence_dup(0).bits.id, io.csr_dup(0).hgatp.vmid, s2xlate = io.csr_dup(0).priv.virt))).asUInt) 970 } 971 } 972 } 973 974 val hfencev_valid = sfence_dup(0).valid && sfence_dup(0).bits.hv 975 when (hfencev_valid) { 976 val l2vmidhit = VecInit(l2vmids.map(_.getOrElse(0.U) === io.csr_dup(2).hgatp.vmid)).asUInt 977 val spvmidhit = VecInit(spvmids.map(_.getOrElse(0.U) === io.csr_dup(0).hgatp.vmid)).asUInt 978 val l2hhit = VecInit(l2h.map(_ === onlyStage1)).asUInt 979 val sphhit = VecInit(sph.map(_ === onlyStage1)).asUInt 980 val l1hhit = VecInit(l1h.flatMap(_.map(_ === onlyStage1))).asUInt 981 val hfencev_vpn = sfence_dup(0).bits.addr(sfence_dup(0).bits.addr.getWidth-1, offLen) 982 when(sfence_dup(0).bits.rs1) { 983 when(sfence_dup(0).bits.rs2) { 984 l1v := l1v & ~l1hhit 985 l2v := l2v & ~(l2hhit & l2vmidhit) 986 spv := spv & ~(sphhit & spvmidhit) 987 }.otherwise { 988 l1v := l1v & (l1g | ~l1hhit) 989 l2v := l2v & ~(~l2g & l2hhit & l2asidhit & l2vmidhit) 990 spv := spv & ~(~spg & sphhit & spasidhit & spvmidhit) 991 } 992 }.otherwise { 993 when(sfence_dup(0).bits.rs2) { 994 spv := spv & ~(sphhit & VecInit(sp.map(_.hit(hfencev_vpn, sfence_dup(0).bits.id, sfence_dup(0).bits.id, io.csr_dup(0).hgatp.vmid, ignoreAsid = true, s2xlate = true.B))).asUInt) 995 }.otherwise { 996 spv := spv & ~(~spg & sphhit & VecInit(sp.map(_.hit(hfencev_vpn, sfence_dup(0).bits.id, sfence_dup(0).bits.id, io.csr_dup(0).hgatp.vmid, s2xlate = true.B))).asUInt) 997 } 998 } 999 } 1000 1001 1002 val hfenceg_valid = sfence_dup(0).valid && sfence_dup(0).bits.hg 1003 when(hfenceg_valid) { 1004 val l2vmidhit = VecInit(l2vmids.map(_.getOrElse(0.U) === sfence_dup(2).bits.id)).asUInt 1005 val spvmidhit = VecInit(spvmids.map(_.getOrElse(0.U) === sfence_dup(0).bits.id)).asUInt 1006 val l2hhit = VecInit(l2h.map(_ === onlyStage2)).asUInt 1007 val sphhit = VecInit(sph.map(_ === onlyStage2)).asUInt 1008 val l1hhit = VecInit(l1h.flatMap(_.map(_ === onlyStage2))).asUInt 1009 val hfenceg_gvpn = (sfence_dup(0).bits.addr << 2)(sfence_dup(0).bits.addr.getWidth - 1, offLen) 1010 when(sfence_dup(0).bits.rs1) { 1011 when(sfence_dup(0).bits.rs2) { 1012 l1v := l1v & ~l1hhit 1013 l2v := l2v & ~l2hhit 1014 spv := spv & ~sphhit 1015 }.otherwise { 1016 l1v := l1v & ~l1hhit 1017 l2v := l2v & ~(l2hhit & l2vmidhit) 1018 spv := spv & ~(sphhit & spvmidhit) 1019 } 1020 }.otherwise { 1021 when(sfence_dup(0).bits.rs2) { 1022 spv := spv & ~(sphhit & VecInit(sp.map(_.hit(hfenceg_gvpn, 0.U, 0.U, sfence_dup(0).bits.id, ignoreAsid = true, s2xlate = false.B))).asUInt) 1023 }.otherwise { 1024 spv := spv & ~(~spg & sphhit & VecInit(sp.map(_.hit(hfenceg_gvpn, 0.U, 0.U, sfence_dup(0).bits.id, ignoreAsid = true, s2xlate = true.B))).asUInt) 1025 } 1026 } 1027 } 1028 1029 if (EnableSv48) { 1030 val l3asidhit = VecInit(l3asids.get.map(_ === sfence_dup(2).bits.id)).asUInt 1031 val l3vmidhit = VecInit(l3vmids.get.map(_.getOrElse(0.U) === io.csr_dup(2).hgatp.vmid)).asUInt 1032 val l3hhit = VecInit(l3h.get.map{a => io.csr_dup(2).priv.virt && a === onlyStage1 || !io.csr_dup(2).priv.virt && a === noS2xlate}).asUInt 1033 1034 when (sfence_valid) { 1035 val l3vmidhit = VecInit(l3vmids.get.map(_.getOrElse(0.U) === io.csr_dup(2).hgatp.vmid)).asUInt 1036 val l3hhit = VecInit(l3h.get.map{a => io.csr_dup(2).priv.virt && a === onlyStage1 || !io.csr_dup(2).priv.virt && a === noS2xlate}).asUInt 1037 val sfence_vpn = sfence_dup(2).bits.addr(sfence_dup(2).bits.addr.getWidth-1, offLen) 1038 1039 when (sfence_dup(2).bits.rs1/*va*/) { 1040 when (sfence_dup(2).bits.rs2) { 1041 // all va && all asid 1042 l3v.map(_ := l3v.get & ~(l3hhit & VecInit(l3vmidhit.asBools.map{a => io.csr_dup(2).priv.virt && a || !io.csr_dup(2).priv.virt}).asUInt)) 1043 } .otherwise { 1044 // all va && specific asid except global 1045 l3v.map(_ := l3v.get & ~(~l3g.get & l3hhit & l3asidhit & VecInit(l3vmidhit.asBools.map{a => io.csr_dup(2).priv.virt && a || !io.csr_dup(2).priv.virt}).asUInt)) 1046 } 1047 } 1048 } 1049 1050 when (hfencev_valid) { 1051 val l3vmidhit = VecInit(l3vmids.get.map(_.getOrElse(0.U) === io.csr_dup(2).hgatp.vmid)).asUInt 1052 val l3hhit = VecInit(l3h.get.map(_ === onlyStage1)).asUInt 1053 val hfencev_vpn = sfence_dup(2).bits.addr(sfence_dup(2).bits.addr.getWidth-1, offLen) 1054 when(sfence_dup(2).bits.rs1) { 1055 when(sfence_dup(2).bits.rs2) { 1056 l3v.map(_ := l3v.get & ~(l3hhit & l3vmidhit)) 1057 }.otherwise { 1058 l3v.map(_ := l3v.get & ~(~l3g.get & l3hhit & l3asidhit & l3vmidhit)) 1059 } 1060 } 1061 } 1062 1063 when (hfenceg_valid) { 1064 val l3vmidhit = VecInit(l3vmids.get.map(_.getOrElse(0.U) === sfence_dup(2).bits.id)).asUInt 1065 val l3hhit = VecInit(l3h.get.map(_ === onlyStage2)).asUInt 1066 val hfenceg_gvpn = (sfence_dup(2).bits.addr << 2)(sfence_dup(2).bits.addr.getWidth - 1, offLen) 1067 when(sfence_dup(2).bits.rs1) { 1068 when(sfence_dup(2).bits.rs2) { 1069 l3v.map(_ := l3v.get & ~l3hhit) 1070 }.otherwise { 1071 l3v.map(_ := l3v.get & ~(l3hhit & l3vmidhit)) 1072 } 1073 } 1074 } 1075 } 1076 1077 def InsideStageConnect(in: DecoupledIO[PtwCacheReq], out: DecoupledIO[PtwCacheReq], inFire: Bool): Unit = { 1078 in.ready := !in.valid || out.ready 1079 out.valid := in.valid 1080 out.bits := in.bits 1081 out.bits.bypassed.zip(in.bits.bypassed).zipWithIndex.map{ case (b, i) => 1082 val bypassed_reg = Reg(Bool()) 1083 val bypassed_wire = refill_bypass(in.bits.req_info.vpn, i, in.bits.req_info.s2xlate) && io.refill.valid 1084 when (inFire) { bypassed_reg := bypassed_wire } 1085 .elsewhen (io.refill.valid) { bypassed_reg := bypassed_reg || bypassed_wire } 1086 1087 b._1 := b._2 || (bypassed_wire || (bypassed_reg && !inFire)) 1088 } 1089 } 1090 1091 // Perf Count 1092 val resp_l0 = resp_res.l0.hit 1093 val resp_sp = resp_res.sp.hit 1094 val resp_l3_pre = if (EnableSv48) Some(resp_res.l3.get.pre) else None 1095 val resp_l2_pre = resp_res.l2.pre 1096 val resp_l1_pre = resp_res.l1.pre 1097 val resp_l0_pre = resp_res.l0.pre 1098 val resp_sp_pre = resp_res.sp.pre 1099 val base_valid_access_0 = !from_pre(io.resp.bits.req_info.source) && io.resp.fire 1100 XSPerfAccumulate("access", base_valid_access_0) 1101 if (EnableSv48) { 1102 XSPerfAccumulate("l3_hit", base_valid_access_0 && io.resp.bits.toFsm.l3Hit.get && !io.resp.bits.toFsm.l2Hit && !io.resp.bits.toFsm.l1Hit && !io.resp.bits.hit) 1103 } 1104 XSPerfAccumulate("l2_hit", base_valid_access_0 && io.resp.bits.toFsm.l2Hit && !io.resp.bits.toFsm.l1Hit && !io.resp.bits.hit) 1105 XSPerfAccumulate("l1_hit", base_valid_access_0 && io.resp.bits.toFsm.l1Hit && !io.resp.bits.hit) 1106 XSPerfAccumulate("l0_hit", base_valid_access_0 && resp_l0) 1107 XSPerfAccumulate("sp_hit", base_valid_access_0 && resp_sp) 1108 XSPerfAccumulate("pte_hit",base_valid_access_0 && io.resp.bits.hit) 1109 1110 if (EnableSv48) { 1111 XSPerfAccumulate("l3_hit_pre", base_valid_access_0 && resp_l3_pre.get && io.resp.bits.toFsm.l3Hit.get && !io.resp.bits.toFsm.l2Hit && !io.resp.bits.toFsm.l1Hit && !io.resp.bits.hit) 1112 } 1113 XSPerfAccumulate("l2_hit_pre", base_valid_access_0 && resp_l2_pre && !io.resp.bits.toFsm.l2Hit && !io.resp.bits.toFsm.l1Hit && !io.resp.bits.hit) 1114 XSPerfAccumulate("l1_hit_pre", base_valid_access_0 && resp_l1_pre && io.resp.bits.toFsm.l1Hit && !io.resp.bits.hit) 1115 XSPerfAccumulate("l0_hit_pre", base_valid_access_0 && resp_l0_pre && resp_l0) 1116 XSPerfAccumulate("sp_hit_pre", base_valid_access_0 && resp_sp_pre && resp_sp) 1117 XSPerfAccumulate("pte_hit_pre",base_valid_access_0 && (resp_l0_pre && resp_l0 || resp_sp_pre && resp_sp) && io.resp.bits.hit) 1118 1119 val base_valid_access_1 = from_pre(io.resp.bits.req_info.source) && io.resp.fire 1120 XSPerfAccumulate("pre_access", base_valid_access_1) 1121 if (EnableSv48) { 1122 XSPerfAccumulate("pre_l3_hit", base_valid_access_1 && io.resp.bits.toFsm.l3Hit.get && !io.resp.bits.toFsm.l2Hit && !io.resp.bits.toFsm.l1Hit && !io.resp.bits.hit) 1123 } 1124 XSPerfAccumulate("pre_l2_hit", base_valid_access_1 && io.resp.bits.toFsm.l2Hit && !io.resp.bits.toFsm.l1Hit && !io.resp.bits.hit) 1125 XSPerfAccumulate("pre_l1_hit", base_valid_access_1 && io.resp.bits.toFsm.l1Hit && !io.resp.bits.hit) 1126 XSPerfAccumulate("pre_l0_hit", base_valid_access_1 && resp_l0) 1127 XSPerfAccumulate("pre_sp_hit", base_valid_access_1 && resp_sp) 1128 XSPerfAccumulate("pre_pte_hit",base_valid_access_1 && io.resp.bits.hit) 1129 1130 if (EnableSv48) { 1131 XSPerfAccumulate("pre_l3_hit_pre", base_valid_access_1 && resp_l3_pre.get && io.resp.bits.toFsm.l3Hit.get && !io.resp.bits.toFsm.l2Hit && !io.resp.bits.toFsm.l1Hit && !io.resp.bits.hit) 1132 } 1133 XSPerfAccumulate("pre_l2_hit_pre", base_valid_access_1 && resp_l2_pre && io.resp.bits.toFsm.l2Hit && !io.resp.bits.toFsm.l1Hit && !io.resp.bits.hit) 1134 XSPerfAccumulate("pre_l1_hit_pre", base_valid_access_1 && resp_l1_pre && io.resp.bits.toFsm.l1Hit && !io.resp.bits.hit) 1135 XSPerfAccumulate("pre_l0_hit_pre", base_valid_access_1 && resp_l0_pre && resp_l0) 1136 XSPerfAccumulate("pre_sp_hit_pre", base_valid_access_1 && resp_sp_pre && resp_sp) 1137 XSPerfAccumulate("pre_pte_hit_pre",base_valid_access_1 && (resp_l0_pre && resp_l0 || resp_sp_pre && resp_sp) && io.resp.bits.hit) 1138 1139 val base_valid_access_2 = stageResp.bits.isFirst && !from_pre(io.resp.bits.req_info.source) && io.resp.fire 1140 XSPerfAccumulate("access_first", base_valid_access_2) 1141 if (EnableSv48) { 1142 XSPerfAccumulate("l3_hit_first", base_valid_access_2 && io.resp.bits.toFsm.l3Hit.get && !io.resp.bits.toFsm.l2Hit && !io.resp.bits.toFsm.l1Hit && !io.resp.bits.hit) 1143 } 1144 XSPerfAccumulate("l2_hit_first", base_valid_access_2 && io.resp.bits.toFsm.l2Hit && !io.resp.bits.toFsm.l1Hit && !io.resp.bits.hit) 1145 XSPerfAccumulate("l1_hit_first", base_valid_access_2 && io.resp.bits.toFsm.l1Hit && !io.resp.bits.hit) 1146 XSPerfAccumulate("l0_hit_first", base_valid_access_2 && resp_l0) 1147 XSPerfAccumulate("sp_hit_first", base_valid_access_2 && resp_sp) 1148 XSPerfAccumulate("pte_hit_first",base_valid_access_2 && io.resp.bits.hit) 1149 1150 if (EnableSv48) { 1151 XSPerfAccumulate("l3_hit_pre_first", base_valid_access_2 && resp_l3_pre.get && io.resp.bits.toFsm.l3Hit.get && !io.resp.bits.toFsm.l2Hit && !io.resp.bits.toFsm.l1Hit && !io.resp.bits.hit) 1152 } 1153 XSPerfAccumulate("l2_hit_pre_first", base_valid_access_2 && resp_l2_pre && io.resp.bits.toFsm.l2Hit && !io.resp.bits.toFsm.l1Hit && !io.resp.bits.hit) 1154 XSPerfAccumulate("l1_hit_pre_first", base_valid_access_2 && resp_l1_pre && io.resp.bits.toFsm.l1Hit && !io.resp.bits.hit) 1155 XSPerfAccumulate("l0_hit_pre_first", base_valid_access_2 && resp_l0_pre && resp_l0) 1156 XSPerfAccumulate("sp_hit_pre_first", base_valid_access_2 && resp_sp_pre && resp_sp) 1157 XSPerfAccumulate("pte_hit_pre_first",base_valid_access_2 && (resp_l0_pre && resp_l0 || resp_sp_pre && resp_sp) && io.resp.bits.hit) 1158 1159 val base_valid_access_3 = stageResp.bits.isFirst && from_pre(io.resp.bits.req_info.source) && io.resp.fire 1160 XSPerfAccumulate("pre_access_first", base_valid_access_3) 1161 if (EnableSv48) { 1162 XSPerfAccumulate("pre_l3_hit_first", base_valid_access_3 && io.resp.bits.toFsm.l3Hit.get && !io.resp.bits.toFsm.l2Hit && !io.resp.bits.toFsm.l1Hit && !io.resp.bits.hit) 1163 } 1164 XSPerfAccumulate("pre_l2_hit_first", base_valid_access_3 && io.resp.bits.toFsm.l2Hit && !io.resp.bits.toFsm.l1Hit && !io.resp.bits.hit) 1165 XSPerfAccumulate("pre_l1_hit_first", base_valid_access_3 && io.resp.bits.toFsm.l1Hit && !io.resp.bits.hit) 1166 XSPerfAccumulate("pre_l0_hit_first", base_valid_access_3 && resp_l0) 1167 XSPerfAccumulate("pre_sp_hit_first", base_valid_access_3 && resp_sp) 1168 XSPerfAccumulate("pre_pte_hit_first", base_valid_access_3 && io.resp.bits.hit) 1169 1170 if (EnableSv48) { 1171 XSPerfAccumulate("pre_l3_hit_pre_first", base_valid_access_3 && resp_l3_pre.get && io.resp.bits.toFsm.l3Hit.get && !io.resp.bits.toFsm.l2Hit && !io.resp.bits.toFsm.l1Hit && !io.resp.bits.hit) 1172 } 1173 XSPerfAccumulate("pre_l2_hit_pre_first", base_valid_access_3 && resp_l2_pre && io.resp.bits.toFsm.l2Hit && !io.resp.bits.toFsm.l1Hit && !io.resp.bits.hit) 1174 XSPerfAccumulate("pre_l1_hit_pre_first", base_valid_access_3 && resp_l1_pre && io.resp.bits.toFsm.l1Hit && !io.resp.bits.hit) 1175 XSPerfAccumulate("pre_l0_hit_pre_first", base_valid_access_3 && resp_l0_pre && resp_l0) 1176 XSPerfAccumulate("pre_sp_hit_pre_first", base_valid_access_3 && resp_sp_pre && resp_sp) 1177 XSPerfAccumulate("pre_pte_hit_pre_first",base_valid_access_3 && (resp_l0_pre && resp_l0 || resp_sp_pre && resp_sp) && io.resp.bits.hit) 1178 1179 XSPerfAccumulate("rwHarzad", io.req.valid && !io.req.ready) 1180 XSPerfAccumulate("out_blocked", io.resp.valid && !io.resp.ready) 1181 if (EnableSv48) { 1182 l3AccessPerf.get.zipWithIndex.map{ case (l, i) => XSPerfAccumulate(s"l3AccessIndex${i}", l) } 1183 } 1184 l2AccessPerf.zipWithIndex.map{ case (l, i) => XSPerfAccumulate(s"l2AccessIndex${i}", l) } 1185 l1AccessPerf.zipWithIndex.map{ case (l, i) => XSPerfAccumulate(s"l1AccessIndex${i}", l) } 1186 l0AccessPerf.zipWithIndex.map{ case (l, i) => XSPerfAccumulate(s"l0AccessIndex${i}", l) } 1187 spAccessPerf.zipWithIndex.map{ case (l, i) => XSPerfAccumulate(s"SPAccessIndex${i}", l) } 1188 if (EnableSv48) { 1189 l3RefillPerf.get.zipWithIndex.map{ case (l, i) => XSPerfAccumulate(s"l3RefillIndex${i}", l) } 1190 } 1191 l2RefillPerf.zipWithIndex.map{ case (l, i) => XSPerfAccumulate(s"l2RefillIndex${i}", l) } 1192 l1RefillPerf.zipWithIndex.map{ case (l, i) => XSPerfAccumulate(s"l1RefillIndex${i}", l) } 1193 l0RefillPerf.zipWithIndex.map{ case (l, i) => XSPerfAccumulate(s"l0RefillIndex${i}", l) } 1194 spRefillPerf.zipWithIndex.map{ case (l, i) => XSPerfAccumulate(s"SPRefillIndex${i}", l) } 1195 1196 if (EnableSv48) { 1197 XSPerfAccumulate("l3Refill", Cat(l3RefillPerf.get).orR) 1198 } 1199 XSPerfAccumulate("l2Refill", Cat(l2RefillPerf).orR) 1200 XSPerfAccumulate("l1Refill", Cat(l1RefillPerf).orR) 1201 XSPerfAccumulate("l0Refill", Cat(l0RefillPerf).orR) 1202 XSPerfAccumulate("spRefill", Cat(spRefillPerf).orR) 1203 if (EnableSv48) { 1204 XSPerfAccumulate("l3Refill_pre", Cat(l3RefillPerf.get).orR && refill_prefetch_dup(0)) 1205 } 1206 XSPerfAccumulate("l2Refill_pre", Cat(l2RefillPerf).orR && refill_prefetch_dup(0)) 1207 XSPerfAccumulate("l1Refill_pre", Cat(l1RefillPerf).orR && refill_prefetch_dup(0)) 1208 XSPerfAccumulate("l0Refill_pre", Cat(l0RefillPerf).orR && refill_prefetch_dup(0)) 1209 XSPerfAccumulate("spRefill_pre", Cat(spRefillPerf).orR && refill_prefetch_dup(0)) 1210 1211 // debug 1212 XSDebug(sfence_dup(0).valid, p"[sfence] original v and g vector:\n") 1213 if (EnableSv48) { 1214 XSDebug(sfence_dup(0).valid, p"[sfence] l3v:${Binary(l3v.get)}\n") 1215 } 1216 XSDebug(sfence_dup(0).valid, p"[sfence] l2v:${Binary(l2v)}\n") 1217 XSDebug(sfence_dup(0).valid, p"[sfence] l1v:${Binary(l1v)}\n") 1218 XSDebug(sfence_dup(0).valid, p"[sfence] l0v:${Binary(l0v)}\n") 1219 XSDebug(sfence_dup(0).valid, p"[sfence] l0g:${Binary(l0g)}\n") 1220 XSDebug(sfence_dup(0).valid, p"[sfence] spv:${Binary(spv)}\n") 1221 XSDebug(RegNext(sfence_dup(0).valid), p"[sfence] new v and g vector:\n") 1222 if (EnableSv48) { 1223 XSDebug(RegNext(sfence_dup(0).valid), p"[sfence] l3v:${Binary(l3v.get)}\n") 1224 } 1225 XSDebug(RegNext(sfence_dup(0).valid), p"[sfence] l2v:${Binary(l2v)}\n") 1226 XSDebug(RegNext(sfence_dup(0).valid), p"[sfence] l1v:${Binary(l1v)}\n") 1227 XSDebug(RegNext(sfence_dup(0).valid), p"[sfence] l0v:${Binary(l0v)}\n") 1228 XSDebug(RegNext(sfence_dup(0).valid), p"[sfence] l0g:${Binary(l0g)}\n") 1229 XSDebug(RegNext(sfence_dup(0).valid), p"[sfence] spv:${Binary(spv)}\n") 1230 1231 val perfEvents = Seq( 1232 ("access ", base_valid_access_0 ), 1233 ("l2_hit ", l2Hit ), 1234 ("l1_hit ", l1Hit ), 1235 ("l0_hit ", l0Hit ), 1236 ("sp_hit ", spHit ), 1237 ("pte_hit ", l0Hit || spHit ), 1238 ("rwHarzad ", io.req.valid && !io.req.ready ), 1239 ("out_blocked ", io.resp.valid && !io.resp.ready), 1240 ) 1241 generatePerfEvent() 1242} 1243