1/*************************************************************************************** 2* Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences 3* Copyright (c) 2020-2021 Peng Cheng Laboratory 4* 5* XiangShan is licensed under Mulan PSL v2. 6* You can use this software according to the terms and conditions of the Mulan PSL v2. 7* You may obtain a copy of Mulan PSL v2 at: 8* http://license.coscl.org.cn/MulanPSL2 9* 10* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, 11* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, 12* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. 13* 14* See the Mulan PSL v2 for more details. 15***************************************************************************************/ 16 17package xiangshan.cache.mmu 18 19import chipsalliance.rocketchip.config.Parameters 20import chisel3._ 21import chisel3.util._ 22import chisel3.internal.naming.chiselName 23import xiangshan._ 24import xiangshan.cache.{HasDCacheParameters, MemoryOpConstants} 25import utils._ 26import utility._ 27import freechips.rocketchip.diplomacy.{LazyModule, LazyModuleImp} 28import freechips.rocketchip.tilelink._ 29 30/* ptw cache caches the page table of all the three layers 31 * ptw cache resp at next cycle 32 * the cache should not be blocked 33 * when miss queue if full, just block req outside 34 */ 35 36class PageCachePerPespBundle(implicit p: Parameters) extends PtwBundle { 37 val hit = Bool() 38 val pre = Bool() 39 val ppn = UInt(ppnLen.W) 40 val perm = new PtePermBundle() 41 val ecc = Bool() 42 val level = UInt(2.W) 43 val v = Bool() 44 45 def apply(hit: Bool, pre: Bool, ppn: UInt, perm: PtePermBundle = 0.U.asTypeOf(new PtePermBundle()), 46 ecc: Bool = false.B, level: UInt = 0.U, valid: Bool = true.B) { 47 this.hit := hit && !ecc 48 this.pre := pre 49 this.ppn := ppn 50 this.perm := perm 51 this.ecc := ecc && hit 52 this.level := level 53 this.v := valid 54 } 55} 56 57class PageCacheRespBundle(implicit p: Parameters) extends PtwBundle { 58 val l1 = new PageCachePerPespBundle 59 val l2 = new PageCachePerPespBundle 60 val l3 = new PageCachePerPespBundle 61 val sp = new PageCachePerPespBundle 62} 63 64class PtwCacheReq(implicit p: Parameters) extends PtwBundle { 65 val req_info = new L2TlbInnerBundle() 66 val isFirst = Bool() 67 val bypassed = Vec(3, Bool()) 68} 69 70class PtwCacheIO()(implicit p: Parameters) extends MMUIOBaseBundle with HasPtwConst { 71 val req = Flipped(DecoupledIO(new PtwCacheReq())) 72 val resp = DecoupledIO(new Bundle { 73 val req_info = new L2TlbInnerBundle() 74 val isFirst = Bool() 75 val hit = Bool() 76 val prefetch = Bool() // is the entry fetched by prefetch 77 val bypassed = Bool() 78 val toFsm = new Bundle { 79 val l1Hit = Bool() 80 val l2Hit = Bool() 81 val ppn = UInt(ppnLen.W) 82 } 83 val toTlb = new PtwEntry(tagLen = vpnLen, hasPerm = true, hasLevel = true) 84 }) 85 val refill = Flipped(ValidIO(new Bundle { 86 val ptes = UInt(blockBits.W) 87 val levelOH = new Bundle { 88 // NOTE: levelOH has (Level+1) bits, each stands for page cache entries 89 val sp = Bool() 90 val l3 = Bool() 91 val l2 = Bool() 92 val l1 = Bool() 93 def apply(levelUInt: UInt, valid: Bool) = { 94 sp := RegNext((levelUInt === 0.U || levelUInt === 1.U) && valid, false.B) 95 l3 := RegNext((levelUInt === 2.U) & valid, false.B) 96 l2 := RegNext((levelUInt === 1.U) & valid, false.B) 97 l1 := RegNext((levelUInt === 0.U) & valid, false.B) 98 } 99 } 100 // duplicate level and sel_pte for each page caches, for better fanout 101 val req_info_dup = Vec(3, new L2TlbInnerBundle()) 102 val level_dup = Vec(3, UInt(log2Up(Level).W)) 103 val sel_pte_dup = Vec(3, UInt(XLEN.W)) 104 })) 105 val sfence_dup = Vec(4, Input(new SfenceBundle())) 106 val csr_dup = Vec(3, Input(new TlbCsrBundle())) 107} 108 109@chiselName 110class PtwCache()(implicit p: Parameters) extends XSModule with HasPtwConst with HasPerfEvents { 111 val io = IO(new PtwCacheIO) 112 113 val ecc = Code.fromString(l2tlbParams.ecc) 114 val l2EntryType = new PTWEntriesWithEcc(ecc, num = PtwL2SectorSize, tagLen = PtwL2TagLen, level = 1, hasPerm = false) 115 val l3EntryType = new PTWEntriesWithEcc(ecc, num = PtwL3SectorSize, tagLen = PtwL3TagLen, level = 2, hasPerm = true) 116 117 // TODO: four caches make the codes dirty, think about how to deal with it 118 119 val sfence_dup = io.sfence_dup 120 val refill = io.refill.bits 121 val refill_prefetch_dup = io.refill.bits.req_info_dup.map(a => from_pre(a.source)) 122 val flush_dup = sfence_dup.zip(io.csr_dup).map(f => f._1.valid || f._2.satp.changed) 123 val flush = flush_dup(0) 124 125 // when refill, refuce to accept new req 126 val rwHarzad = if (sramSinglePort) io.refill.valid else false.B 127 128 // handle hand signal and req_info 129 // TODO: replace with FlushableQueue 130 val stageReq = Wire(Decoupled(new PtwCacheReq())) // enq stage & read page cache valid 131 val stageDelay = Wire(Vec(2, Decoupled(new PtwCacheReq()))) // page cache resp 132 val stageCheck = Wire(Vec(2, Decoupled(new PtwCacheReq()))) // check hit & check ecc 133 val stageResp = Wire(Decoupled(new PtwCacheReq())) // deq stage 134 135 val stageDelay_valid_1cycle = OneCycleValid(stageReq.fire, flush) // catch ram data 136 val stageCheck_valid_1cycle = OneCycleValid(stageDelay(1).fire, flush) // replace & perf counter 137 val stageResp_valid_1cycle_dup = Wire(Vec(2, Bool())) 138 stageResp_valid_1cycle_dup.map(_ := OneCycleValid(stageCheck(1).fire, flush)) // ecc flush 139 140 stageReq <> io.req 141 PipelineConnect(stageReq, stageDelay(0), stageDelay(1).ready, flush, rwHarzad) 142 InsideStageConnect(stageDelay(0), stageDelay(1), stageDelay_valid_1cycle) 143 PipelineConnect(stageDelay(1), stageCheck(0), stageCheck(1).ready, flush) 144 InsideStageConnect(stageCheck(0), stageCheck(1), stageCheck_valid_1cycle) 145 PipelineConnect(stageCheck(1), stageResp, io.resp.ready, flush) 146 stageResp.ready := !stageResp.valid || io.resp.ready 147 148 // l1: level 0 non-leaf pte 149 val l1 = Reg(Vec(l2tlbParams.l1Size, new PtwEntry(tagLen = PtwL1TagLen))) 150 val l1v = RegInit(0.U(l2tlbParams.l1Size.W)) 151 val l1g = Reg(UInt(l2tlbParams.l1Size.W)) 152 val l1asids = Reg(Vec(l2tlbParams.l1Size, UInt(AsidLength.W))) 153 154 // l2: level 1 non-leaf pte 155 val l2 = Module(new SRAMTemplate( 156 l2EntryType, 157 set = l2tlbParams.l2nSets, 158 way = l2tlbParams.l2nWays, 159 singlePort = sramSinglePort 160 )) 161 val l2v = RegInit(0.U((l2tlbParams.l2nSets * l2tlbParams.l2nWays).W)) 162 val l2g = Reg(UInt((l2tlbParams.l2nSets * l2tlbParams.l2nWays).W)) 163 val l2asids = Reg(Vec(l2tlbParams.l2nSets, Vec(l2tlbParams.l2nWays, UInt(AsidLength.W)))) 164 def getl2vSet(vpn: UInt) = { 165 require(log2Up(l2tlbParams.l2nWays) == log2Down(l2tlbParams.l2nWays)) 166 val set = genPtwL2SetIdx(vpn) 167 require(set.getWidth == log2Up(l2tlbParams.l2nSets)) 168 val l2vVec = l2v.asTypeOf(Vec(l2tlbParams.l2nSets, UInt(l2tlbParams.l2nWays.W))) 169 l2vVec(set) 170 } 171 def getl2asidSet(vpn: UInt) = { 172 require(log2Up(l2tlbParams.l2nWays) == log2Down(l2tlbParams.l2nWays)) 173 val set = genPtwL2SetIdx(vpn) 174 require(set.getWidth == log2Up(l2tlbParams.l2nSets)) 175 l2asids(set) 176 } 177 178 // l3: level 2 leaf pte of 4KB pages 179 val l3 = Module(new SRAMTemplate( 180 l3EntryType, 181 set = l2tlbParams.l3nSets, 182 way = l2tlbParams.l3nWays, 183 singlePort = sramSinglePort 184 )) 185 val l3v = RegInit(0.U((l2tlbParams.l3nSets * l2tlbParams.l3nWays).W)) 186 val l3g = Reg(UInt((l2tlbParams.l3nSets * l2tlbParams.l3nWays).W)) 187 val l3asids = Reg(Vec(l2tlbParams.l3nSets, Vec(l2tlbParams.l3nWays, UInt(AsidLength.W)))) 188 def getl3vSet(vpn: UInt) = { 189 require(log2Up(l2tlbParams.l3nWays) == log2Down(l2tlbParams.l3nWays)) 190 val set = genPtwL3SetIdx(vpn) 191 require(set.getWidth == log2Up(l2tlbParams.l3nSets)) 192 val l3vVec = l3v.asTypeOf(Vec(l2tlbParams.l3nSets, UInt(l2tlbParams.l3nWays.W))) 193 l3vVec(set) 194 } 195 def getl3asidSet(vpn: UInt) = { 196 require(log2Up(l2tlbParams.l3nWays) == log2Down(l2tlbParams.l3nWays)) 197 val set = genPtwL3SetIdx(vpn) 198 require(set.getWidth == log2Up(l2tlbParams.l3nSets)) 199 l3asids(set) 200 } 201 202 // sp: level 0/1 leaf pte of 1GB/2MB super pages 203 val sp = Reg(Vec(l2tlbParams.spSize, new PtwEntry(tagLen = SPTagLen, hasPerm = true, hasLevel = true))) 204 val spv = RegInit(0.U(l2tlbParams.spSize.W)) 205 val spg = Reg(UInt(l2tlbParams.spSize.W)) 206 val spasids = Reg(Vec(l2tlbParams.spSize, UInt(AsidLength.W))) 207 208 // Access Perf 209 val l1AccessPerf = Wire(Vec(l2tlbParams.l1Size, Bool())) 210 val l2AccessPerf = Wire(Vec(l2tlbParams.l2nWays, Bool())) 211 val l3AccessPerf = Wire(Vec(l2tlbParams.l3nWays, Bool())) 212 val spAccessPerf = Wire(Vec(l2tlbParams.spSize, Bool())) 213 l1AccessPerf.map(_ := false.B) 214 l2AccessPerf.map(_ := false.B) 215 l3AccessPerf.map(_ := false.B) 216 spAccessPerf.map(_ := false.B) 217 218 219 220 def vpn_match(vpn1: UInt, vpn2: UInt, level: Int) = { 221 vpn1(vpnnLen*3-1, vpnnLen*(2-level)+3) === vpn2(vpnnLen*3-1, vpnnLen*(2-level)+3) 222 } 223 // NOTE: not actually bypassed, just check if hit, re-access the page cache 224 def refill_bypass(vpn: UInt, level: Int) = { 225 io.refill.valid && (level.U === io.refill.bits.level_dup(0)) && vpn_match(io.refill.bits.req_info_dup(0).vpn, vpn, level), 226 } 227 228 // l1 229 val ptwl1replace = ReplacementPolicy.fromString(l2tlbParams.l1Replacer, l2tlbParams.l1Size) 230 val (l1Hit, l1HitPPN, l1Pre) = { 231 val hitVecT = l1.zipWithIndex.map { case (e, i) => e.hit(stageReq.bits.req_info.vpn, io.csr_dup(0).satp.asid) && l1v(i) } 232 val hitVec = hitVecT.map(RegEnable(_, stageReq.fire)) 233 234 // stageDelay, but check for l1 235 val hitPPN = DataHoldBypass(ParallelPriorityMux(hitVec zip l1.map(_.ppn)), stageDelay_valid_1cycle) 236 val hitPre = DataHoldBypass(ParallelPriorityMux(hitVec zip l1.map(_.prefetch)), stageDelay_valid_1cycle) 237 val hit = DataHoldBypass(ParallelOR(hitVec), stageDelay_valid_1cycle) 238 239 when (hit && stageDelay_valid_1cycle) { ptwl1replace.access(OHToUInt(hitVec)) } 240 241 l1AccessPerf.zip(hitVec).map{ case (l, h) => l := h && stageDelay_valid_1cycle} 242 for (i <- 0 until l2tlbParams.l1Size) { 243 XSDebug(stageReq.fire, p"[l1] l1(${i.U}) ${l1(i)} hit:${l1(i).hit(stageReq.bits.req_info.vpn, io.csr_dup(0).satp.asid)}\n") 244 } 245 XSDebug(stageReq.fire, p"[l1] l1v:${Binary(l1v)} hitVecT:${Binary(VecInit(hitVecT).asUInt)}\n") 246 XSDebug(stageDelay(0).valid, p"[l1] l1Hit:${hit} l1HitPPN:0x${Hexadecimal(hitPPN)} hitVec:${VecInit(hitVec).asUInt}\n") 247 248 VecInit(hitVecT).suggestName(s"l1_hitVecT") 249 VecInit(hitVec).suggestName(s"l1_hitVec") 250 251 // synchronize with other entries with RegEnable 252 (RegEnable(hit, stageDelay(1).fire), 253 RegEnable(hitPPN, stageDelay(1).fire), 254 RegEnable(hitPre, stageDelay(1).fire)) 255 } 256 257 // l2 258 val ptwl2replace = ReplacementPolicy.fromString(l2tlbParams.l2Replacer,l2tlbParams.l2nWays,l2tlbParams.l2nSets) 259 val (l2Hit, l2HitPPN, l2Pre, l2eccError) = { 260 val ridx = genPtwL2SetIdx(stageReq.bits.req_info.vpn) 261 l2.io.r.req.valid := stageReq.fire 262 l2.io.r.req.bits.apply(setIdx = ridx) 263 val vVec_req = getl2vSet(stageReq.bits.req_info.vpn) 264 265 // delay one cycle after sram read 266 val delay_vpn = stageDelay(0).bits.req_info.vpn 267 val data_resp = DataHoldBypass(l2.io.r.resp.data, stageDelay_valid_1cycle) 268 val vVec_delay = RegEnable(vVec_req, stageReq.fire) 269 val hitVec_delay = VecInit(data_resp.zip(vVec_delay.asBools).map { case (wayData, v) => 270 wayData.entries.hit(delay_vpn, io.csr_dup(1).satp.asid) && v }) 271 272 // check hit and ecc 273 val check_vpn = stageCheck(0).bits.req_info.vpn 274 val ramDatas = RegEnable(data_resp, stageDelay(1).fire) 275 val vVec = RegEnable(vVec_delay, stageDelay(1).fire).asBools() 276 277 val hitVec = RegEnable(hitVec_delay, stageDelay(1).fire) 278 val hitWayEntry = ParallelPriorityMux(hitVec zip ramDatas) 279 val hitWayData = hitWayEntry.entries 280 val hit = ParallelOR(hitVec) 281 val hitWay = ParallelPriorityMux(hitVec zip (0 until l2tlbParams.l2nWays).map(_.U(log2Up(l2tlbParams.l2nWays).W))) 282 val eccError = hitWayEntry.decode() 283 284 ridx.suggestName(s"l2_ridx") 285 ramDatas.suggestName(s"l2_ramDatas") 286 hitVec.suggestName(s"l2_hitVec") 287 hitWayData.suggestName(s"l2_hitWayData") 288 hitWay.suggestName(s"l2_hitWay") 289 290 when (hit && stageCheck_valid_1cycle) { ptwl2replace.access(genPtwL2SetIdx(check_vpn), hitWay) } 291 292 l2AccessPerf.zip(hitVec).map{ case (l, h) => l := h && stageCheck_valid_1cycle } 293 XSDebug(stageDelay_valid_1cycle, p"[l2] ridx:0x${Hexadecimal(ridx)}\n") 294 for (i <- 0 until l2tlbParams.l2nWays) { 295 XSDebug(stageCheck_valid_1cycle, p"[l2] ramDatas(${i.U}) ${ramDatas(i)} l2v:${vVec(i)} hit:${hit}\n") 296 } 297 XSDebug(stageCheck_valid_1cycle, p"[l2] l2Hit:${hit} l2HitPPN:0x${Hexadecimal(hitWayData.ppns(genPtwL2SectorIdx(check_vpn)))} hitVec:${Binary(hitVec.asUInt)} hitWay:${hitWay} vidx:${vVec}\n") 298 299 (hit, hitWayData.ppns(genPtwL2SectorIdx(check_vpn)), hitWayData.prefetch, eccError) 300 } 301 302 // l3 303 val ptwl3replace = ReplacementPolicy.fromString(l2tlbParams.l3Replacer,l2tlbParams.l3nWays,l2tlbParams.l3nSets) 304 val (l3Hit, l3HitData, l3Pre, l3eccError) = { 305 val ridx = genPtwL3SetIdx(stageReq.bits.req_info.vpn) 306 l3.io.r.req.valid := stageReq.fire 307 l3.io.r.req.bits.apply(setIdx = ridx) 308 val vVec_req = getl3vSet(stageReq.bits.req_info.vpn) 309 310 // delay one cycle after sram read 311 val delay_vpn = stageDelay(0).bits.req_info.vpn 312 val data_resp = DataHoldBypass(l3.io.r.resp.data, stageDelay_valid_1cycle) 313 val vVec_delay = RegEnable(vVec_req, stageReq.fire) 314 val hitVec_delay = VecInit(data_resp.zip(vVec_delay.asBools).map { case (wayData, v) => 315 wayData.entries.hit(delay_vpn, io.csr_dup(2).satp.asid) && v }) 316 317 // check hit and ecc 318 val check_vpn = stageCheck(0).bits.req_info.vpn 319 val ramDatas = RegEnable(data_resp, stageDelay(1).fire) 320 val vVec = RegEnable(vVec_delay, stageDelay(1).fire).asBools() 321 322 val hitVec = RegEnable(hitVec_delay, stageDelay(1).fire) 323 val hitWayEntry = ParallelPriorityMux(hitVec zip ramDatas) 324 val hitWayData = hitWayEntry.entries 325 val hitWayEcc = hitWayEntry.ecc 326 val hit = ParallelOR(hitVec) 327 val hitWay = ParallelPriorityMux(hitVec zip (0 until l2tlbParams.l3nWays).map(_.U(log2Up(l2tlbParams.l3nWays).W))) 328 val eccError = hitWayEntry.decode() 329 330 when (hit && stageCheck_valid_1cycle) { ptwl3replace.access(genPtwL3SetIdx(check_vpn), hitWay) } 331 332 l3AccessPerf.zip(hitVec).map{ case (l, h) => l := h && stageCheck_valid_1cycle } 333 XSDebug(stageReq.fire, p"[l3] ridx:0x${Hexadecimal(ridx)}\n") 334 for (i <- 0 until l2tlbParams.l3nWays) { 335 XSDebug(stageCheck_valid_1cycle, p"[l3] ramDatas(${i.U}) ${ramDatas(i)} l3v:${vVec(i)} hit:${hitVec(i)}\n") 336 } 337 XSDebug(stageCheck_valid_1cycle, p"[l3] l3Hit:${hit} l3HitData:${hitWayData} hitVec:${Binary(hitVec.asUInt)} hitWay:${hitWay} v:${vVec}\n") 338 339 ridx.suggestName(s"l3_ridx") 340 ramDatas.suggestName(s"l3_ramDatas") 341 hitVec.suggestName(s"l3_hitVec") 342 hitWay.suggestName(s"l3_hitWay") 343 344 (hit, hitWayData, hitWayData.prefetch, eccError) 345 } 346 val l3HitPPN = l3HitData.ppns(genPtwL3SectorIdx(stageCheck(0).bits.req_info.vpn)) 347 val l3HitPerm = l3HitData.perms.getOrElse(0.U.asTypeOf(Vec(PtwL3SectorSize, new PtePermBundle)))(genPtwL3SectorIdx(stageCheck(0).bits.req_info.vpn)) 348 val l3HitValid = l3HitData.vs(genPtwL3SectorIdx(stageCheck(0).bits.req_info.vpn)) 349 350 // super page 351 val spreplace = ReplacementPolicy.fromString(l2tlbParams.spReplacer, l2tlbParams.spSize) 352 val (spHit, spHitData, spPre, spValid) = { 353 val hitVecT = sp.zipWithIndex.map { case (e, i) => e.hit(stageReq.bits.req_info.vpn, io.csr_dup(0).satp.asid) && spv(i) } 354 val hitVec = hitVecT.map(RegEnable(_, stageReq.fire)) 355 val hitData = ParallelPriorityMux(hitVec zip sp) 356 val hit = ParallelOR(hitVec) 357 358 when (hit && stageDelay_valid_1cycle) { spreplace.access(OHToUInt(hitVec)) } 359 360 spAccessPerf.zip(hitVec).map{ case (s, h) => s := h && stageDelay_valid_1cycle } 361 for (i <- 0 until l2tlbParams.spSize) { 362 XSDebug(stageReq.fire, p"[sp] sp(${i.U}) ${sp(i)} hit:${sp(i).hit(stageReq.bits.req_info.vpn, io.csr_dup(0).satp.asid)} spv:${spv(i)}\n") 363 } 364 XSDebug(stageDelay_valid_1cycle, p"[sp] spHit:${hit} spHitData:${hitData} hitVec:${Binary(VecInit(hitVec).asUInt)}\n") 365 366 VecInit(hitVecT).suggestName(s"sp_hitVecT") 367 VecInit(hitVec).suggestName(s"sp_hitVec") 368 369 (RegEnable(hit, stageDelay(1).fire), 370 RegEnable(hitData, stageDelay(1).fire), 371 RegEnable(hitData.prefetch, stageDelay(1).fire), 372 RegEnable(hitData.v, stageDelay(1).fire())) 373 } 374 val spHitPerm = spHitData.perm.getOrElse(0.U.asTypeOf(new PtePermBundle)) 375 val spHitLevel = spHitData.level.getOrElse(0.U) 376 377 val check_res = Wire(new PageCacheRespBundle) 378 check_res.l1.apply(l1Hit, l1Pre, l1HitPPN) 379 check_res.l2.apply(l2Hit, l2Pre, l2HitPPN, ecc = l2eccError) 380 check_res.l3.apply(l3Hit, l3Pre, l3HitPPN, l3HitPerm, l3eccError, valid = l3HitValid) 381 check_res.sp.apply(spHit, spPre, spHitData.ppn, spHitPerm, false.B, spHitLevel, spValid) 382 383 val resp_res = Reg(new PageCacheRespBundle) 384 when (stageCheck(1).fire) { resp_res := check_res } 385 386 // stageResp bypass 387 val bypassed = Wire(Vec(3, Bool())) 388 bypassed.indices.foreach(i => 389 bypassed(i) := stageResp.bits.bypassed(i) || 390 ValidHoldBypass(refill_bypass(stageResp.bits.req_info.vpn, i), 391 OneCycleValid(stageCheck(1).fire, false.B) || io.refill.valid) 392 ) 393 394 io.resp.bits.req_info := stageResp.bits.req_info 395 io.resp.bits.isFirst := stageResp.bits.isFirst 396 io.resp.bits.hit := resp_res.l3.hit || resp_res.sp.hit 397 io.resp.bits.bypassed := bypassed(2) || (bypassed(1) && !resp_res.l2.hit) || (bypassed(0) && !resp_res.l1.hit) 398 io.resp.bits.prefetch := resp_res.l3.pre && resp_res.l3.hit || resp_res.sp.pre && resp_res.sp.hit 399 io.resp.bits.toFsm.l1Hit := resp_res.l1.hit 400 io.resp.bits.toFsm.l2Hit := resp_res.l2.hit 401 io.resp.bits.toFsm.ppn := Mux(resp_res.l2.hit, resp_res.l2.ppn, resp_res.l1.ppn) 402 io.resp.bits.toTlb.tag := stageResp.bits.req_info.vpn 403 io.resp.bits.toTlb.asid := io.csr_dup(0).satp.asid // DontCare 404 io.resp.bits.toTlb.ppn := Mux(resp_res.l3.hit, resp_res.l3.ppn, resp_res.sp.ppn) 405 io.resp.bits.toTlb.perm.map(_ := Mux(resp_res.l3.hit, resp_res.l3.perm, resp_res.sp.perm)) 406 io.resp.bits.toTlb.level.map(_ := Mux(resp_res.l3.hit, 2.U, resp_res.sp.level)) 407 io.resp.bits.toTlb.prefetch := from_pre(stageResp.bits.req_info.source) 408 io.resp.bits.toTlb.v := Mux(resp_res.sp.hit, resp_res.sp.v, resp_res.l3.v) 409 io.resp.valid := stageResp.valid 410 XSError(stageResp.valid && resp_res.l3.hit && resp_res.sp.hit, "normal page and super page both hit") 411 XSError(stageResp.valid && io.resp.bits.hit && bypassed(2), "page cache, bypassed but hit") 412 413 // refill Perf 414 val l1RefillPerf = Wire(Vec(l2tlbParams.l1Size, Bool())) 415 val l2RefillPerf = Wire(Vec(l2tlbParams.l2nWays, Bool())) 416 val l3RefillPerf = Wire(Vec(l2tlbParams.l3nWays, Bool())) 417 val spRefillPerf = Wire(Vec(l2tlbParams.spSize, Bool())) 418 l1RefillPerf.map(_ := false.B) 419 l2RefillPerf.map(_ := false.B) 420 l3RefillPerf.map(_ := false.B) 421 spRefillPerf.map(_ := false.B) 422 423 // refill 424 l2.io.w.req <> DontCare 425 l3.io.w.req <> DontCare 426 l2.io.w.req.valid := false.B 427 l3.io.w.req.valid := false.B 428 429 val memRdata = refill.ptes 430 val memPtes = (0 until (l2tlbParams.blockBytes/(XLEN/8))).map(i => memRdata((i+1)*XLEN-1, i*XLEN).asTypeOf(new PteBundle)) 431 val memSelData = io.refill.bits.sel_pte_dup 432 val memPte = memSelData.map(a => a.asTypeOf(new PteBundle)) 433 434 // TODO: handle sfenceLatch outsize 435 when (!flush_dup(0) && refill.levelOH.l1 && !memPte(0).isLeaf() && !memPte(0).isPf(refill.level_dup(0))) { 436 // val refillIdx = LFSR64()(log2Up(l2tlbParams.l1Size)-1,0) // TODO: may be LRU 437 val refillIdx = replaceWrapper(l1v, ptwl1replace.way) 438 refillIdx.suggestName(s"PtwL1RefillIdx") 439 val rfOH = UIntToOH(refillIdx) 440 l1(refillIdx).refill( 441 refill.req_info_dup(0).vpn, 442 io.csr_dup(0).satp.asid, 443 memSelData(0), 444 0.U, 445 refill_prefetch_dup(0) 446 ) 447 ptwl1replace.access(refillIdx) 448 l1v := l1v | rfOH 449 l1g := (l1g & ~rfOH) | Mux(memPte(0).perm.g, rfOH, 0.U) 450 451 for (i <- 0 until l2tlbParams.l1Size) { 452 l1RefillPerf(i) := i.U === refillIdx 453 } 454 455 XSDebug(p"[l1 refill] refillIdx:${refillIdx} refillEntry:${l1(refillIdx).genPtwEntry(refill.req_info_dup(0).vpn, io.csr_dup(0).satp.asid, memSelData(0), 0.U, prefetch = refill_prefetch_dup(0))}\n") 456 XSDebug(p"[l1 refill] l1v:${Binary(l1v)}->${Binary(l1v | rfOH)} l1g:${Binary(l1g)}->${Binary((l1g & ~rfOH) | Mux(memPte(0).perm.g, rfOH, 0.U))}\n") 457 458 refillIdx.suggestName(s"l1_refillIdx") 459 rfOH.suggestName(s"l1_rfOH") 460 } 461 462 when (!flush_dup(1) && refill.levelOH.l2 && !memPte(1).isLeaf() && !memPte(1).isPf(refill.level_dup(1))) { 463 val refillIdx = genPtwL2SetIdx(refill.req_info_dup(1).vpn) 464 val victimWay = replaceWrapper(getl2vSet(refill.req_info_dup(1).vpn), ptwl2replace.way(refillIdx)) 465 val victimWayOH = UIntToOH(victimWay) 466 val rfvOH = UIntToOH(Cat(refillIdx, victimWay)) 467 val wdata = Wire(l2EntryType) 468 wdata.gen( 469 vpn = refill.req_info_dup(1).vpn, 470 asid = io.csr_dup(1).satp.asid, 471 data = memRdata, 472 levelUInt = 1.U, 473 refill_prefetch_dup(1) 474 ) 475 l2.io.w.apply( 476 valid = true.B, 477 setIdx = refillIdx, 478 data = wdata, 479 waymask = victimWayOH 480 ) 481 ptwl2replace.access(refillIdx, victimWay) 482 l2v := l2v | rfvOH 483 l2g := l2g & ~rfvOH | Mux(Cat(memPtes.map(_.perm.g)).andR, rfvOH, 0.U) 484 485 for (i <- 0 until l2tlbParams.l2nWays) { 486 l2RefillPerf(i) := i.U === victimWay 487 } 488 489 XSDebug(p"[l2 refill] refillIdx:0x${Hexadecimal(refillIdx)} victimWay:${victimWay} victimWayOH:${Binary(victimWayOH)} rfvOH(in UInt):${Cat(refillIdx, victimWay)}\n") 490 XSDebug(p"[l2 refill] refilldata:0x${wdata}\n") 491 XSDebug(p"[l2 refill] l2v:${Binary(l2v)} -> ${Binary(l2v | rfvOH)}\n") 492 XSDebug(p"[l2 refill] l2g:${Binary(l2g)} -> ${Binary(l2g & ~rfvOH | Mux(Cat(memPtes.map(_.perm.g)).andR, rfvOH, 0.U))}\n") 493 494 refillIdx.suggestName(s"l2_refillIdx") 495 victimWay.suggestName(s"l2_victimWay") 496 victimWayOH.suggestName(s"l2_victimWayOH") 497 rfvOH.suggestName(s"l2_rfvOH") 498 } 499 500 when (!flush_dup(2) && refill.levelOH.l3) { 501 val refillIdx = genPtwL3SetIdx(refill.req_info_dup(2).vpn) 502 val victimWay = replaceWrapper(getl3vSet(refill.req_info_dup(2).vpn), ptwl3replace.way(refillIdx)) 503 val victimWayOH = UIntToOH(victimWay) 504 val rfvOH = UIntToOH(Cat(refillIdx, victimWay)) 505 val wdata = Wire(l3EntryType) 506 wdata.gen( 507 vpn = refill.req_info_dup(2).vpn, 508 asid = io.csr_dup(2).satp.asid, 509 data = memRdata, 510 levelUInt = 2.U, 511 refill_prefetch_dup(2) 512 ) 513 l3.io.w.apply( 514 valid = true.B, 515 setIdx = refillIdx, 516 data = wdata, 517 waymask = victimWayOH 518 ) 519 ptwl3replace.access(refillIdx, victimWay) 520 l3v := l3v | rfvOH 521 l3g := l3g & ~rfvOH | Mux(Cat(memPtes.map(_.perm.g)).andR, rfvOH, 0.U) 522 523 for (i <- 0 until l2tlbParams.l3nWays) { 524 l3RefillPerf(i) := i.U === victimWay 525 } 526 527 XSDebug(p"[l3 refill] refillIdx:0x${Hexadecimal(refillIdx)} victimWay:${victimWay} victimWayOH:${Binary(victimWayOH)} rfvOH(in UInt):${Cat(refillIdx, victimWay)}\n") 528 XSDebug(p"[l3 refill] refilldata:0x${wdata}\n") 529 XSDebug(p"[l3 refill] l3v:${Binary(l3v)} -> ${Binary(l3v | rfvOH)}\n") 530 XSDebug(p"[l3 refill] l3g:${Binary(l3g)} -> ${Binary(l3g & ~rfvOH | Mux(Cat(memPtes.map(_.perm.g)).andR, rfvOH, 0.U))}\n") 531 532 refillIdx.suggestName(s"l3_refillIdx") 533 victimWay.suggestName(s"l3_victimWay") 534 victimWayOH.suggestName(s"l3_victimWayOH") 535 rfvOH.suggestName(s"l3_rfvOH") 536 } 537 538 539 // misc entries: super & invalid 540 when (!flush_dup(0) && refill.levelOH.sp && (memPte(0).isLeaf() || memPte(0).isPf(refill.level_dup(0)))) { 541 val refillIdx = spreplace.way// LFSR64()(log2Up(l2tlbParams.spSize)-1,0) // TODO: may be LRU 542 val rfOH = UIntToOH(refillIdx) 543 sp(refillIdx).refill( 544 refill.req_info_dup(0).vpn, 545 io.csr_dup(0).satp.asid, 546 memSelData(0), 547 refill.level_dup(2), 548 refill_prefetch_dup(0), 549 !memPte(0).isPf(refill.level_dup(0)), 550 ) 551 spreplace.access(refillIdx) 552 spv := spv | rfOH 553 spg := spg & ~rfOH | Mux(memPte(0).perm.g, rfOH, 0.U) 554 555 for (i <- 0 until l2tlbParams.spSize) { 556 spRefillPerf(i) := i.U === refillIdx 557 } 558 559 XSDebug(p"[sp refill] refillIdx:${refillIdx} refillEntry:${sp(refillIdx).genPtwEntry(refill.req_info_dup(0).vpn, io.csr_dup(0).satp.asid, memSelData(0), refill.level_dup(0), refill_prefetch_dup(0))}\n") 560 XSDebug(p"[sp refill] spv:${Binary(spv)}->${Binary(spv | rfOH)} spg:${Binary(spg)}->${Binary(spg & ~rfOH | Mux(memPte(0).perm.g, rfOH, 0.U))}\n") 561 562 refillIdx.suggestName(s"sp_refillIdx") 563 rfOH.suggestName(s"sp_rfOH") 564 } 565 566 val l2eccFlush = resp_res.l2.ecc && stageResp_valid_1cycle_dup(0) // RegNext(l2eccError, init = false.B) 567 val l3eccFlush = resp_res.l3.ecc && stageResp_valid_1cycle_dup(1) // RegNext(l3eccError, init = false.B) 568 val eccVpn = stageResp.bits.req_info.vpn 569 570 XSError(l2eccFlush, "l2tlb.cache.l2 ecc error. Should not happen at sim stage") 571 XSError(l3eccFlush, "l2tlb.cache.l3 ecc error. Should not happen at sim stage") 572 when (l2eccFlush) { 573 val flushSetIdxOH = UIntToOH(genPtwL2SetIdx(eccVpn)) 574 val flushMask = VecInit(flushSetIdxOH.asBools.map { a => Fill(l2tlbParams.l2nWays, a.asUInt) }).asUInt 575 l2v := l2v & ~flushMask 576 l2g := l2g & ~flushMask 577 } 578 579 when (l3eccFlush) { 580 val flushSetIdxOH = UIntToOH(genPtwL3SetIdx(eccVpn)) 581 val flushMask = VecInit(flushSetIdxOH.asBools.map { a => Fill(l2tlbParams.l3nWays, a.asUInt) }).asUInt 582 l3v := l3v & ~flushMask 583 l3g := l3g & ~flushMask 584 } 585 586 // sfence 587 when (sfence_dup(3).valid) { 588 val sfence_vpn = sfence_dup(3).bits.addr(sfence_dup(3).bits.addr.getWidth-1, offLen) 589 590 when (sfence_dup(3).bits.rs1/*va*/) { 591 when (sfence_dup(3).bits.rs2) { 592 // all va && all asid 593 l3v := 0.U 594 } .otherwise { 595 // all va && specific asid except global 596 l3v := l3v & l3g 597 } 598 } .otherwise { 599 // val flushMask = UIntToOH(genTlbL2Idx(sfence.bits.addr(sfence.bits.addr.getWidth-1, offLen))) 600 val flushSetIdxOH = UIntToOH(genPtwL3SetIdx(sfence_vpn)) 601 // val flushMask = VecInit(flushSetIdxOH.asBools.map(Fill(l2tlbParams.l3nWays, _.asUInt))).asUInt 602 val flushMask = VecInit(flushSetIdxOH.asBools.map { a => Fill(l2tlbParams.l3nWays, a.asUInt) }).asUInt 603 flushSetIdxOH.suggestName(s"sfence_nrs1_flushSetIdxOH") 604 flushMask.suggestName(s"sfence_nrs1_flushMask") 605 606 when (sfence_dup(3).bits.rs2) { 607 // specific leaf of addr && all asid 608 l3v := l3v & ~flushMask 609 } .otherwise { 610 // specific leaf of addr && specific asid 611 l3v := l3v & (~flushMask | l3g) 612 } 613 } 614 } 615 616 when (sfence_dup(0).valid) { 617 val l1asidhit = VecInit(l1asids.map(_ === sfence_dup(0).bits.asid)).asUInt 618 val spasidhit = VecInit(spasids.map(_ === sfence_dup(0).bits.asid)).asUInt 619 val sfence_vpn = sfence_dup(0).bits.addr(sfence_dup(0).bits.addr.getWidth-1, offLen) 620 621 when (sfence_dup(0).bits.rs1/*va*/) { 622 when (sfence_dup(0).bits.rs2) { 623 // all va && all asid 624 l1v := 0.U 625 l2v := 0.U 626 spv := 0.U 627 } .otherwise { 628 // all va && specific asid except global 629 630 l1v := l1v & (~l1asidhit | l1g) 631 l2v := l2v & l2g 632 spv := spv & (~spasidhit | spg) 633 } 634 } .otherwise { 635 // val flushMask = UIntToOH(genTlbL2Idx(sfence.bits.addr(sfence.bits.addr.getWidth-1, offLen))) 636 val flushSetIdxOH = UIntToOH(genPtwL3SetIdx(sfence_vpn)) 637 // val flushMask = VecInit(flushSetIdxOH.asBools.map(Fill(l2tlbParams.l3nWays, _.asUInt))).asUInt 638 val flushMask = VecInit(flushSetIdxOH.asBools.map { a => Fill(l2tlbParams.l3nWays, a.asUInt) }).asUInt 639 flushSetIdxOH.suggestName(s"sfence_nrs1_flushSetIdxOH") 640 flushMask.suggestName(s"sfence_nrs1_flushMask") 641 642 when (sfence_dup(0).bits.rs2) { 643 // specific leaf of addr && all asid 644 spv := spv & (~VecInit(sp.map(_.hit(sfence_vpn, sfence_dup(0).bits.asid, ignoreAsid = true))).asUInt) 645 } .otherwise { 646 // specific leaf of addr && specific asid 647 spv := spv & (~VecInit(sp.map(_.hit(sfence_vpn, sfence_dup(0).bits.asid))).asUInt | spg) 648 } 649 } 650 } 651 652 def InsideStageConnect(in: DecoupledIO[PtwCacheReq], out: DecoupledIO[PtwCacheReq], inFire: Bool): Unit = { 653 in.ready := !in.valid || out.ready 654 out.valid := in.valid 655 out.bits := in.bits 656 out.bits.bypassed.zip(in.bits.bypassed).zipWithIndex.map{ case (b, i) => 657 val bypassed_reg = Reg(Bool()) 658 val bypassed_wire = refill_bypass(in.bits.req_info.vpn, i) && io.refill.valid 659 when (inFire) { bypassed_reg := bypassed_wire } 660 .elsewhen (io.refill.valid) { bypassed_reg := bypassed_reg || bypassed_wire } 661 662 b._1 := b._2 || (bypassed_wire || (bypassed_reg && !inFire)) 663 } 664 } 665 666 // Perf Count 667 val resp_l3 = resp_res.l3.hit 668 val resp_sp = resp_res.sp.hit 669 val resp_l1_pre = resp_res.l1.pre 670 val resp_l2_pre = resp_res.l2.pre 671 val resp_l3_pre = resp_res.l3.pre 672 val resp_sp_pre = resp_res.sp.pre 673 val base_valid_access_0 = !from_pre(io.resp.bits.req_info.source) && io.resp.fire() 674 XSPerfAccumulate("access", base_valid_access_0) 675 XSPerfAccumulate("l1_hit", base_valid_access_0 && io.resp.bits.toFsm.l1Hit && !io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 676 XSPerfAccumulate("l2_hit", base_valid_access_0 && io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 677 XSPerfAccumulate("l3_hit", base_valid_access_0 && resp_l3) 678 XSPerfAccumulate("sp_hit", base_valid_access_0 && resp_sp) 679 XSPerfAccumulate("pte_hit",base_valid_access_0 && io.resp.bits.hit) 680 681 XSPerfAccumulate("l1_hit_pre", base_valid_access_0 && resp_l1_pre && io.resp.bits.toFsm.l1Hit && !io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 682 XSPerfAccumulate("l2_hit_pre", base_valid_access_0 && resp_l2_pre && io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 683 XSPerfAccumulate("l3_hit_pre", base_valid_access_0 && resp_l3_pre && resp_l3) 684 XSPerfAccumulate("sp_hit_pre", base_valid_access_0 && resp_sp_pre && resp_sp) 685 XSPerfAccumulate("pte_hit_pre",base_valid_access_0 && (resp_l3_pre && resp_l3 || resp_sp_pre && resp_sp) && io.resp.bits.hit) 686 687 val base_valid_access_1 = from_pre(io.resp.bits.req_info.source) && io.resp.fire() 688 XSPerfAccumulate("pre_access", base_valid_access_1) 689 XSPerfAccumulate("pre_l1_hit", base_valid_access_1 && io.resp.bits.toFsm.l1Hit && !io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 690 XSPerfAccumulate("pre_l2_hit", base_valid_access_1 && io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 691 XSPerfAccumulate("pre_l3_hit", base_valid_access_1 && resp_l3) 692 XSPerfAccumulate("pre_sp_hit", base_valid_access_1 && resp_sp) 693 XSPerfAccumulate("pre_pte_hit",base_valid_access_1 && io.resp.bits.hit) 694 695 XSPerfAccumulate("pre_l1_hit_pre", base_valid_access_1 && resp_l1_pre && io.resp.bits.toFsm.l1Hit && !io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 696 XSPerfAccumulate("pre_l2_hit_pre", base_valid_access_1 && resp_l2_pre && io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 697 XSPerfAccumulate("pre_l3_hit_pre", base_valid_access_1 && resp_l3_pre && resp_l3) 698 XSPerfAccumulate("pre_sp_hit_pre", base_valid_access_1 && resp_sp_pre && resp_sp) 699 XSPerfAccumulate("pre_pte_hit_pre",base_valid_access_1 && (resp_l3_pre && resp_l3 || resp_sp_pre && resp_sp) && io.resp.bits.hit) 700 701 val base_valid_access_2 = stageResp.bits.isFirst && !from_pre(io.resp.bits.req_info.source) && io.resp.fire() 702 XSPerfAccumulate("access_first", base_valid_access_2) 703 XSPerfAccumulate("l1_hit_first", base_valid_access_2 && io.resp.bits.toFsm.l1Hit && !io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 704 XSPerfAccumulate("l2_hit_first", base_valid_access_2 && io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 705 XSPerfAccumulate("l3_hit_first", base_valid_access_2 && resp_l3) 706 XSPerfAccumulate("sp_hit_first", base_valid_access_2 && resp_sp) 707 XSPerfAccumulate("pte_hit_first",base_valid_access_2 && io.resp.bits.hit) 708 709 XSPerfAccumulate("l1_hit_pre_first", base_valid_access_2 && resp_l1_pre && io.resp.bits.toFsm.l1Hit && !io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 710 XSPerfAccumulate("l2_hit_pre_first", base_valid_access_2 && resp_l2_pre && io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 711 XSPerfAccumulate("l3_hit_pre_first", base_valid_access_2 && resp_l3_pre && resp_l3) 712 XSPerfAccumulate("sp_hit_pre_first", base_valid_access_2 && resp_sp_pre && resp_sp) 713 XSPerfAccumulate("pte_hit_pre_first",base_valid_access_2 && (resp_l3_pre && resp_l3 || resp_sp_pre && resp_sp) && io.resp.bits.hit) 714 715 val base_valid_access_3 = stageResp.bits.isFirst && from_pre(io.resp.bits.req_info.source) && io.resp.fire() 716 XSPerfAccumulate("pre_access_first", base_valid_access_3) 717 XSPerfAccumulate("pre_l1_hit_first", base_valid_access_3 && io.resp.bits.toFsm.l1Hit && !io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 718 XSPerfAccumulate("pre_l2_hit_first", base_valid_access_3 && io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 719 XSPerfAccumulate("pre_l3_hit_first", base_valid_access_3 && resp_l3) 720 XSPerfAccumulate("pre_sp_hit_first", base_valid_access_3 && resp_sp) 721 XSPerfAccumulate("pre_pte_hit_first", base_valid_access_3 && io.resp.bits.hit) 722 723 XSPerfAccumulate("pre_l1_hit_pre_first", base_valid_access_3 && resp_l1_pre && io.resp.bits.toFsm.l1Hit && !io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 724 XSPerfAccumulate("pre_l2_hit_pre_first", base_valid_access_3 && resp_l2_pre && io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 725 XSPerfAccumulate("pre_l3_hit_pre_first", base_valid_access_3 && resp_l3_pre && resp_l3) 726 XSPerfAccumulate("pre_sp_hit_pre_first", base_valid_access_3 && resp_sp_pre && resp_sp) 727 XSPerfAccumulate("pre_pte_hit_pre_first",base_valid_access_3 && (resp_l3_pre && resp_l3 || resp_sp_pre && resp_sp) && io.resp.bits.hit) 728 729 XSPerfAccumulate("rwHarzad", io.req.valid && !io.req.ready) 730 XSPerfAccumulate("out_blocked", io.resp.valid && !io.resp.ready) 731 l1AccessPerf.zipWithIndex.map{ case (l, i) => XSPerfAccumulate(s"L1AccessIndex${i}", l) } 732 l2AccessPerf.zipWithIndex.map{ case (l, i) => XSPerfAccumulate(s"L2AccessIndex${i}", l) } 733 l3AccessPerf.zipWithIndex.map{ case (l, i) => XSPerfAccumulate(s"L3AccessIndex${i}", l) } 734 spAccessPerf.zipWithIndex.map{ case (l, i) => XSPerfAccumulate(s"SPAccessIndex${i}", l) } 735 l1RefillPerf.zipWithIndex.map{ case (l, i) => XSPerfAccumulate(s"L1RefillIndex${i}", l) } 736 l2RefillPerf.zipWithIndex.map{ case (l, i) => XSPerfAccumulate(s"L2RefillIndex${i}", l) } 737 l3RefillPerf.zipWithIndex.map{ case (l, i) => XSPerfAccumulate(s"L3RefillIndex${i}", l) } 738 spRefillPerf.zipWithIndex.map{ case (l, i) => XSPerfAccumulate(s"SPRefillIndex${i}", l) } 739 740 XSPerfAccumulate("l1Refill", Cat(l1RefillPerf).orR) 741 XSPerfAccumulate("l2Refill", Cat(l2RefillPerf).orR) 742 XSPerfAccumulate("l3Refill", Cat(l3RefillPerf).orR) 743 XSPerfAccumulate("spRefill", Cat(spRefillPerf).orR) 744 XSPerfAccumulate("l1Refill_pre", Cat(l1RefillPerf).orR && refill_prefetch_dup(0)) 745 XSPerfAccumulate("l2Refill_pre", Cat(l2RefillPerf).orR && refill_prefetch_dup(0)) 746 XSPerfAccumulate("l3Refill_pre", Cat(l3RefillPerf).orR && refill_prefetch_dup(0)) 747 XSPerfAccumulate("spRefill_pre", Cat(spRefillPerf).orR && refill_prefetch_dup(0)) 748 749 // debug 750 XSDebug(sfence_dup(0).valid, p"[sfence] original v and g vector:\n") 751 XSDebug(sfence_dup(0).valid, p"[sfence] l1v:${Binary(l1v)}\n") 752 XSDebug(sfence_dup(0).valid, p"[sfence] l2v:${Binary(l2v)}\n") 753 XSDebug(sfence_dup(0).valid, p"[sfence] l3v:${Binary(l3v)}\n") 754 XSDebug(sfence_dup(0).valid, p"[sfence] l3g:${Binary(l3g)}\n") 755 XSDebug(sfence_dup(0).valid, p"[sfence] spv:${Binary(spv)}\n") 756 XSDebug(RegNext(sfence_dup(0).valid), p"[sfence] new v and g vector:\n") 757 XSDebug(RegNext(sfence_dup(0).valid), p"[sfence] l1v:${Binary(l1v)}\n") 758 XSDebug(RegNext(sfence_dup(0).valid), p"[sfence] l2v:${Binary(l2v)}\n") 759 XSDebug(RegNext(sfence_dup(0).valid), p"[sfence] l3v:${Binary(l3v)}\n") 760 XSDebug(RegNext(sfence_dup(0).valid), p"[sfence] l3g:${Binary(l3g)}\n") 761 XSDebug(RegNext(sfence_dup(0).valid), p"[sfence] spv:${Binary(spv)}\n") 762 763 val perfEvents = Seq( 764 ("access ", base_valid_access_0 ), 765 ("l1_hit ", l1Hit ), 766 ("l2_hit ", l2Hit ), 767 ("l3_hit ", l3Hit ), 768 ("sp_hit ", spHit ), 769 ("pte_hit ", l3Hit || spHit ), 770 ("rwHarzad ", io.req.valid && !io.req.ready ), 771 ("out_blocked ", io.resp.valid && !io.resp.ready), 772 ) 773 generatePerfEvent() 774} 775