1/*************************************************************************************** 2* Copyright (c) 2024 Beijing Institute of Open Source Chip (BOSC) 3* Copyright (c) 2020-2024 Institute of Computing Technology, Chinese Academy of Sciences 4* Copyright (c) 2020-2021 Peng Cheng Laboratory 5* 6* XiangShan is licensed under Mulan PSL v2. 7* You can use this software according to the terms and conditions of the Mulan PSL v2. 8* You may obtain a copy of Mulan PSL v2 at: 9* http://license.coscl.org.cn/MulanPSL2 10* 11* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, 12* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, 13* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. 14* 15* See the Mulan PSL v2 for more details. 16***************************************************************************************/ 17 18package xiangshan.cache.mmu 19 20import org.chipsalliance.cde.config.Parameters 21import chisel3._ 22import chisel3.util._ 23import xiangshan._ 24import xiangshan.cache.{HasDCacheParameters, MemoryOpConstants} 25import utils._ 26import utility._ 27import freechips.rocketchip.diplomacy.{LazyModule, LazyModuleImp} 28import freechips.rocketchip.tilelink._ 29 30/* ptw cache caches the page table of all the three layers 31 * ptw cache resp at next cycle 32 * the cache should not be blocked 33 * when miss queue if full, just block req outside 34 */ 35 36class PageCachePerPespBundle(implicit p: Parameters) extends PtwBundle { 37 val hit = Bool() 38 val pre = Bool() 39 val ppn = UInt(gvpnLen.W) 40 val perm = new PtePermBundle() 41 val ecc = Bool() 42 val level = UInt(2.W) 43 val v = Bool() 44 45 def apply(hit: Bool, pre: Bool, ppn: UInt, perm: PtePermBundle = 0.U.asTypeOf(new PtePermBundle()), 46 ecc: Bool = false.B, level: UInt = 0.U, valid: Bool = true.B): Unit = { 47 this.hit := hit && !ecc 48 this.pre := pre 49 this.ppn := ppn 50 this.perm := perm 51 this.ecc := ecc && hit 52 this.level := level 53 this.v := valid 54 } 55} 56 57class PageCacheMergePespBundle(implicit p: Parameters) extends PtwBundle { 58 assert(tlbcontiguous == 8, "Only support tlbcontiguous = 8!") 59 val hit = Bool() 60 val pre = Bool() 61 val ppn = Vec(tlbcontiguous, UInt(gvpnLen.W)) 62 val perm = Vec(tlbcontiguous, new PtePermBundle()) 63 val ecc = Bool() 64 val level = UInt(2.W) 65 val v = Vec(tlbcontiguous, Bool()) 66 67 def apply(hit: Bool, pre: Bool, ppn: Vec[UInt], perm: Vec[PtePermBundle] = Vec(tlbcontiguous, 0.U.asTypeOf(new PtePermBundle())), 68 ecc: Bool = false.B, level: UInt = 0.U, valid: Vec[Bool] = Vec(tlbcontiguous, true.B)): Unit = { 69 this.hit := hit && !ecc 70 this.pre := pre 71 this.ppn := ppn 72 this.perm := perm 73 this.ecc := ecc && hit 74 this.level := level 75 this.v := valid 76 } 77} 78 79class PageCacheRespBundle(implicit p: Parameters) extends PtwBundle { 80 val l1 = new PageCachePerPespBundle 81 val l2 = new PageCachePerPespBundle 82 val l3 = new PageCacheMergePespBundle 83 val sp = new PageCachePerPespBundle 84} 85 86class PtwCacheReq(implicit p: Parameters) extends PtwBundle { 87 val req_info = new L2TlbInnerBundle() 88 val isFirst = Bool() 89 val bypassed = Vec(3, Bool()) 90 val isHptwReq = Bool() 91 val hptwId = UInt(log2Up(l2tlbParams.llptwsize).W) 92} 93 94class PtwCacheIO()(implicit p: Parameters) extends MMUIOBaseBundle with HasPtwConst { 95 val req = Flipped(DecoupledIO(new PtwCacheReq())) 96 val resp = DecoupledIO(new Bundle { 97 val req_info = new L2TlbInnerBundle() 98 val isFirst = Bool() 99 val hit = Bool() 100 val prefetch = Bool() // is the entry fetched by prefetch 101 val bypassed = Bool() 102 val toFsm = new Bundle { 103 val l1Hit = Bool() 104 val l2Hit = Bool() 105 val ppn = UInt(gvpnLen.W) 106 val stage1Hit = Bool() // find stage 1 pte in cache, but need to search stage 2 pte in cache at PTW 107 } 108 val stage1 = new PtwMergeResp() 109 val isHptwReq = Bool() 110 val toHptw = new Bundle { 111 val l1Hit = Bool() 112 val l2Hit = Bool() 113 val ppn = UInt(ppnLen.W) 114 val id = UInt(log2Up(l2tlbParams.llptwsize).W) 115 val resp = new HptwResp() // used if hit 116 val bypassed = Bool() 117 } 118 }) 119 val refill = Flipped(ValidIO(new Bundle { 120 val ptes = UInt(blockBits.W) 121 val levelOH = new Bundle { 122 // NOTE: levelOH has (Level+1) bits, each stands for page cache entries 123 val sp = Bool() 124 val l3 = Bool() 125 val l2 = Bool() 126 val l1 = Bool() 127 def apply(levelUInt: UInt, valid: Bool) = { 128 sp := GatedValidRegNext((levelUInt === 0.U || levelUInt === 1.U) && valid, false.B) 129 l3 := GatedValidRegNext((levelUInt === 2.U) & valid, false.B) 130 l2 := GatedValidRegNext((levelUInt === 1.U) & valid, false.B) 131 l1 := GatedValidRegNext((levelUInt === 0.U) & valid, false.B) 132 } 133 } 134 // duplicate level and sel_pte for each page caches, for better fanout 135 val req_info_dup = Vec(3, new L2TlbInnerBundle()) 136 val level_dup = Vec(3, UInt(log2Up(Level).W)) 137 val sel_pte_dup = Vec(3, UInt(XLEN.W)) 138 })) 139 val sfence_dup = Vec(4, Input(new SfenceBundle())) 140 val csr_dup = Vec(3, Input(new TlbCsrBundle())) 141} 142 143class PtwCache()(implicit p: Parameters) extends XSModule with HasPtwConst with HasPerfEvents { 144 val io = IO(new PtwCacheIO) 145 val ecc = Code.fromString(l2tlbParams.ecc) 146 val l2EntryType = new PTWEntriesWithEcc(ecc, num = PtwL2SectorSize, tagLen = PtwL2TagLen, level = 1, hasPerm = false, hasReservedBitforMbist = true) 147 val l3EntryType = new PTWEntriesWithEcc(ecc, num = PtwL3SectorSize, tagLen = PtwL3TagLen, level = 2, hasPerm = true) 148 149 // TODO: four caches make the codes dirty, think about how to deal with it 150 151 val sfence_dup = io.sfence_dup 152 val refill = io.refill.bits 153 val refill_prefetch_dup = io.refill.bits.req_info_dup.map(a => from_pre(a.source)) 154 val flush_dup = sfence_dup.zip(io.csr_dup).map(f => f._1.valid || f._2.satp.changed || f._2.vsatp.changed || f._2.hgatp.changed) 155 val flush = flush_dup(0) 156 157 // when refill, refuce to accept new req 158 val rwHarzad = if (sramSinglePort) io.refill.valid else false.B 159 160 // handle hand signal and req_info 161 // TODO: replace with FlushableQueue 162 val stageReq = Wire(Decoupled(new PtwCacheReq())) // enq stage & read page cache valid 163 val stageDelay = Wire(Vec(2, Decoupled(new PtwCacheReq()))) // page cache resp 164 val stageCheck = Wire(Vec(2, Decoupled(new PtwCacheReq()))) // check hit & check ecc 165 val stageResp = Wire(Decoupled(new PtwCacheReq())) // deq stage 166 167 val stageDelay_valid_1cycle = OneCycleValid(stageReq.fire, flush) // catch ram data 168 val stageCheck_valid_1cycle = OneCycleValid(stageDelay(1).fire, flush) // replace & perf counter 169 val stageResp_valid_1cycle_dup = Wire(Vec(2, Bool())) 170 stageResp_valid_1cycle_dup.map(_ := OneCycleValid(stageCheck(1).fire, flush)) // ecc flush 171 172 stageReq <> io.req 173 PipelineConnect(stageReq, stageDelay(0), stageDelay(1).ready, flush, rwHarzad) 174 InsideStageConnect(stageDelay(0), stageDelay(1), stageDelay_valid_1cycle) 175 PipelineConnect(stageDelay(1), stageCheck(0), stageCheck(1).ready, flush) 176 InsideStageConnect(stageCheck(0), stageCheck(1), stageCheck_valid_1cycle) 177 PipelineConnect(stageCheck(1), stageResp, io.resp.ready, flush) 178 stageResp.ready := !stageResp.valid || io.resp.ready 179 180 // l1: level 0 non-leaf pte 181 val l1 = Reg(Vec(l2tlbParams.l1Size, new PtwEntry(tagLen = PtwL1TagLen))) 182 val l1v = RegInit(0.U(l2tlbParams.l1Size.W)) 183 val l1g = Reg(UInt(l2tlbParams.l1Size.W)) 184 val l1asids = l1.map(_.asid) 185 val l1vmids = l1.map(_.vmid) 186 val l1h = Reg(Vec(l2tlbParams.l1Size, UInt(2.W))) 187 188 // l2: level 1 non-leaf pte 189 val l2 = Module(new SRAMTemplate( 190 l2EntryType, 191 set = l2tlbParams.l2nSets, 192 way = l2tlbParams.l2nWays, 193 singlePort = sramSinglePort 194 )) 195 val l2v = RegInit(0.U((l2tlbParams.l2nSets * l2tlbParams.l2nWays).W)) 196 val l2g = Reg(UInt((l2tlbParams.l2nSets * l2tlbParams.l2nWays).W)) 197 val l2h = Reg(Vec(l2tlbParams.l2nSets, Vec(l2tlbParams.l2nWays, UInt(2.W)))) 198 def getl2vSet(vpn: UInt) = { 199 require(log2Up(l2tlbParams.l2nWays) == log2Down(l2tlbParams.l2nWays)) 200 val set = genPtwL2SetIdx(vpn) 201 require(set.getWidth == log2Up(l2tlbParams.l2nSets)) 202 val l2vVec = l2v.asTypeOf(Vec(l2tlbParams.l2nSets, UInt(l2tlbParams.l2nWays.W))) 203 l2vVec(set) 204 } 205 def getl2hSet(vpn: UInt) = { 206 require(log2Up(l2tlbParams.l2nWays) == log2Down(l2tlbParams.l2nWays)) 207 val set = genPtwL2SetIdx(vpn) 208 require(set.getWidth == log2Up(l2tlbParams.l2nSets)) 209 l2h(set) 210 } 211 212 213 214 // l3: level 2 leaf pte of 4KB pages 215 val l3 = Module(new SRAMTemplate( 216 l3EntryType, 217 set = l2tlbParams.l3nSets, 218 way = l2tlbParams.l3nWays, 219 singlePort = sramSinglePort 220 )) 221 val l3v = RegInit(0.U((l2tlbParams.l3nSets * l2tlbParams.l3nWays).W)) 222 val l3g = Reg(UInt((l2tlbParams.l3nSets * l2tlbParams.l3nWays).W)) 223 val l3h = Reg(Vec(l2tlbParams.l3nSets, Vec(l2tlbParams.l3nWays, UInt(2.W)))) 224 def getl3vSet(vpn: UInt) = { 225 require(log2Up(l2tlbParams.l3nWays) == log2Down(l2tlbParams.l3nWays)) 226 val set = genPtwL3SetIdx(vpn) 227 require(set.getWidth == log2Up(l2tlbParams.l3nSets)) 228 val l3vVec = l3v.asTypeOf(Vec(l2tlbParams.l3nSets, UInt(l2tlbParams.l3nWays.W))) 229 l3vVec(set) 230 } 231 def getl3hSet(vpn: UInt) = { 232 require(log2Up(l2tlbParams.l3nWays) == log2Down(l2tlbParams.l3nWays)) 233 val set = genPtwL3SetIdx(vpn) 234 require(set.getWidth == log2Up(l2tlbParams.l3nSets)) 235 l3h(set) 236 } 237 238 // sp: level 0/1 leaf pte of 1GB/2MB super pages 239 val sp = Reg(Vec(l2tlbParams.spSize, new PtwEntry(tagLen = SPTagLen, hasPerm = true, hasLevel = true))) 240 val spv = RegInit(0.U(l2tlbParams.spSize.W)) 241 val spg = Reg(UInt(l2tlbParams.spSize.W)) 242 val spasids = sp.map(_.asid) 243 val spvmids = sp.map(_.vmid) 244 val sph = Reg(Vec(l2tlbParams.spSize, UInt(2.W))) 245 246 // Access Perf 247 val l1AccessPerf = Wire(Vec(l2tlbParams.l1Size, Bool())) 248 val l2AccessPerf = Wire(Vec(l2tlbParams.l2nWays, Bool())) 249 val l3AccessPerf = Wire(Vec(l2tlbParams.l3nWays, Bool())) 250 val spAccessPerf = Wire(Vec(l2tlbParams.spSize, Bool())) 251 l1AccessPerf.map(_ := false.B) 252 l2AccessPerf.map(_ := false.B) 253 l3AccessPerf.map(_ := false.B) 254 spAccessPerf.map(_ := false.B) 255 256 257 258 def vpn_match(vpn1: UInt, vpn2: UInt, level: Int) = { 259 (vpn1(vpnLen-1, vpnnLen*(2-level)+3) === vpn2(vpnLen-1, vpnnLen*(2-level)+3)) 260 } 261 // NOTE: not actually bypassed, just check if hit, re-access the page cache 262 def refill_bypass(vpn: UInt, level: Int, h_search: UInt) = { 263 val change_h = MuxLookup(h_search, noS2xlate)(Seq( 264 allStage -> onlyStage1, 265 onlyStage1 -> onlyStage1, 266 onlyStage2 -> onlyStage2 267 )) 268 val refill_vpn = io.refill.bits.req_info_dup(0).vpn 269 io.refill.valid && (level.U === io.refill.bits.level_dup(0)) && vpn_match(refill_vpn, vpn, level) && change_h === io.refill.bits.req_info_dup(0).s2xlate 270 } 271 272 val vpn_search = stageReq.bits.req_info.vpn 273 val h_search = MuxLookup(stageReq.bits.req_info.s2xlate, noS2xlate)(Seq( 274 allStage -> onlyStage1, 275 onlyStage1 -> onlyStage1, 276 onlyStage2 -> onlyStage2 277 )) 278 // l1 279 val ptwl1replace = ReplacementPolicy.fromString(l2tlbParams.l1Replacer, l2tlbParams.l1Size) 280 val (l1Hit, l1HitPPN, l1Pre) = { 281 val hitVecT = l1.zipWithIndex.map { 282 case (e, i) => (e.hit(vpn_search, io.csr_dup(0).satp.asid, io.csr_dup(0).vsatp.asid, io.csr_dup(0).hgatp.asid, s2xlate = h_search =/= noS2xlate) 283 && l1v(i) && h_search === l1h(i)) 284 } 285 val hitVec = hitVecT.map(RegEnable(_, stageReq.fire)) 286 287 // stageDelay, but check for l1 288 val hitPPN = DataHoldBypass(ParallelPriorityMux(hitVec zip l1.map(_.ppn)), stageDelay_valid_1cycle) 289 val hitPre = DataHoldBypass(ParallelPriorityMux(hitVec zip l1.map(_.prefetch)), stageDelay_valid_1cycle) 290 val hit = DataHoldBypass(ParallelOR(hitVec), stageDelay_valid_1cycle) 291 292 when (hit && stageDelay_valid_1cycle) { ptwl1replace.access(OHToUInt(hitVec)) } 293 294 l1AccessPerf.zip(hitVec).map{ case (l, h) => l := h && stageDelay_valid_1cycle} 295 for (i <- 0 until l2tlbParams.l1Size) { 296 XSDebug(stageReq.fire, p"[l1] l1(${i.U}) ${l1(i)} hit:${l1(i).hit(vpn_search, io.csr_dup(0).satp.asid, io.csr_dup(0).vsatp.asid, io.csr_dup(0).hgatp.asid, s2xlate = h_search =/= noS2xlate)}\n") 297 } 298 XSDebug(stageReq.fire, p"[l1] l1v:${Binary(l1v)} hitVecT:${Binary(VecInit(hitVecT).asUInt)}\n") 299 XSDebug(stageDelay(0).valid, p"[l1] l1Hit:${hit} l1HitPPN:0x${Hexadecimal(hitPPN)} hitVec:${VecInit(hitVec).asUInt}\n") 300 301 VecInit(hitVecT).suggestName(s"l1_hitVecT") 302 VecInit(hitVec).suggestName(s"l1_hitVec") 303 304 // synchronize with other entries with RegEnable 305 (RegEnable(hit, stageDelay(1).fire), 306 RegEnable(hitPPN, stageDelay(1).fire), 307 RegEnable(hitPre, stageDelay(1).fire)) 308 } 309 310 // l2 311 val ptwl2replace = ReplacementPolicy.fromString(l2tlbParams.l2Replacer,l2tlbParams.l2nWays,l2tlbParams.l2nSets) 312 val (l2Hit, l2HitPPN, l2Pre, l2eccError) = { 313 val ridx = genPtwL2SetIdx(vpn_search) 314 l2.io.r.req.valid := stageReq.fire 315 l2.io.r.req.bits.apply(setIdx = ridx) 316 val vVec_req = getl2vSet(vpn_search) 317 val hVec_req = getl2hSet(vpn_search) 318 319 // delay one cycle after sram read 320 val delay_vpn = stageDelay(0).bits.req_info.vpn 321 val delay_h = MuxLookup(stageDelay(0).bits.req_info.s2xlate, noS2xlate)(Seq( 322 allStage -> onlyStage1, 323 onlyStage1 -> onlyStage1, 324 onlyStage2 -> onlyStage2 325 )) 326 val data_resp = DataHoldBypass(l2.io.r.resp.data, stageDelay_valid_1cycle) 327 val vVec_delay = RegEnable(vVec_req, stageReq.fire) 328 val hVec_delay = RegEnable(hVec_req, stageReq.fire) 329 val hitVec_delay = VecInit(data_resp.zip(vVec_delay.asBools).zip(hVec_delay).map { case ((wayData, v), h) => 330 wayData.entries.hit(delay_vpn, io.csr_dup(1).satp.asid, io.csr_dup(1).vsatp.asid, io.csr_dup(1).hgatp.asid, s2xlate = delay_h =/= noS2xlate) && v && (delay_h === h)}) 331 332 // check hit and ecc 333 val check_vpn = stageCheck(0).bits.req_info.vpn 334 val ramDatas = RegEnable(data_resp, stageDelay(1).fire) 335 val vVec = RegEnable(vVec_delay, stageDelay(1).fire).asBools 336 337 val hitVec = RegEnable(hitVec_delay, stageDelay(1).fire) 338 val hitWayEntry = ParallelPriorityMux(hitVec zip ramDatas) 339 val hitWayData = hitWayEntry.entries 340 val hit = ParallelOR(hitVec) 341 val hitWay = ParallelPriorityMux(hitVec zip (0 until l2tlbParams.l2nWays).map(_.U(log2Up(l2tlbParams.l2nWays).W))) 342 val eccError = WireInit(false.B) 343 if (l2tlbParams.enablePTWECC) { 344 eccError := hitWayEntry.decode() 345 } else { 346 eccError := false.B 347 } 348 349 ridx.suggestName(s"l2_ridx") 350 ramDatas.suggestName(s"l2_ramDatas") 351 hitVec.suggestName(s"l2_hitVec") 352 hitWayData.suggestName(s"l2_hitWayData") 353 hitWay.suggestName(s"l2_hitWay") 354 355 when (hit && stageCheck_valid_1cycle) { ptwl2replace.access(genPtwL2SetIdx(check_vpn), hitWay) } 356 357 l2AccessPerf.zip(hitVec).map{ case (l, h) => l := h && stageCheck_valid_1cycle } 358 XSDebug(stageDelay_valid_1cycle, p"[l2] ridx:0x${Hexadecimal(ridx)}\n") 359 for (i <- 0 until l2tlbParams.l2nWays) { 360 XSDebug(stageCheck_valid_1cycle, p"[l2] ramDatas(${i.U}) ${ramDatas(i)} l2v:${vVec(i)} hit:${hit}\n") 361 } 362 XSDebug(stageCheck_valid_1cycle, p"[l2] l2Hit:${hit} l2HitPPN:0x${Hexadecimal(hitWayData.ppns(genPtwL2SectorIdx(check_vpn)))} hitVec:${Binary(hitVec.asUInt)} hitWay:${hitWay} vidx:${vVec}\n") 363 364 (hit, hitWayData.ppns(genPtwL2SectorIdx(check_vpn)), hitWayData.prefetch, eccError) 365 } 366 367 // l3 368 val ptwl3replace = ReplacementPolicy.fromString(l2tlbParams.l3Replacer,l2tlbParams.l3nWays,l2tlbParams.l3nSets) 369 val (l3Hit, l3HitData, l3Pre, l3eccError) = { 370 val ridx = genPtwL3SetIdx(vpn_search) 371 l3.io.r.req.valid := stageReq.fire 372 l3.io.r.req.bits.apply(setIdx = ridx) 373 val vVec_req = getl3vSet(vpn_search) 374 val hVec_req = getl3hSet(vpn_search) 375 376 // delay one cycle after sram read 377 val delay_vpn = stageDelay(0).bits.req_info.vpn 378 val delay_h = MuxLookup(stageDelay(0).bits.req_info.s2xlate, noS2xlate)(Seq( 379 allStage -> onlyStage1, 380 onlyStage1 -> onlyStage1, 381 onlyStage2 -> onlyStage2 382 )) 383 val data_resp = DataHoldBypass(l3.io.r.resp.data, stageDelay_valid_1cycle) 384 val vVec_delay = RegEnable(vVec_req, stageReq.fire) 385 val hVec_delay = RegEnable(hVec_req, stageReq.fire) 386 val hitVec_delay = VecInit(data_resp.zip(vVec_delay.asBools).zip(hVec_delay).map { case ((wayData, v), h) => 387 wayData.entries.hit(delay_vpn, io.csr_dup(2).satp.asid, io.csr_dup(2).vsatp.asid, io.csr_dup(2).hgatp.asid, s2xlate = delay_h =/= noS2xlate) && v && (delay_h === h)}) 388 389 // check hit and ecc 390 val check_vpn = stageCheck(0).bits.req_info.vpn 391 val ramDatas = RegEnable(data_resp, stageDelay(1).fire) 392 val vVec = RegEnable(vVec_delay, stageDelay(1).fire).asBools 393 394 val hitVec = RegEnable(hitVec_delay, stageDelay(1).fire) 395 val hitWayEntry = ParallelPriorityMux(hitVec zip ramDatas) 396 val hitWayData = hitWayEntry.entries 397 val hitWayEcc = hitWayEntry.ecc 398 val hit = ParallelOR(hitVec) 399 val hitWay = ParallelPriorityMux(hitVec zip (0 until l2tlbParams.l3nWays).map(_.U(log2Up(l2tlbParams.l3nWays).W))) 400 val eccError = WireInit(false.B) 401 if (l2tlbParams.enablePTWECC) { 402 eccError := hitWayEntry.decode() 403 } else { 404 eccError := false.B 405 } 406 407 when (hit && stageCheck_valid_1cycle) { ptwl3replace.access(genPtwL3SetIdx(check_vpn), hitWay) } 408 409 l3AccessPerf.zip(hitVec).map{ case (l, h) => l := h && stageCheck_valid_1cycle } 410 XSDebug(stageReq.fire, p"[l3] ridx:0x${Hexadecimal(ridx)}\n") 411 for (i <- 0 until l2tlbParams.l3nWays) { 412 XSDebug(stageCheck_valid_1cycle, p"[l3] ramDatas(${i.U}) ${ramDatas(i)} l3v:${vVec(i)} hit:${hitVec(i)}\n") 413 } 414 XSDebug(stageCheck_valid_1cycle, p"[l3] l3Hit:${hit} l3HitData:${hitWayData} hitVec:${Binary(hitVec.asUInt)} hitWay:${hitWay} v:${vVec}\n") 415 416 ridx.suggestName(s"l3_ridx") 417 ramDatas.suggestName(s"l3_ramDatas") 418 hitVec.suggestName(s"l3_hitVec") 419 hitWay.suggestName(s"l3_hitWay") 420 421 (hit, hitWayData, hitWayData.prefetch, eccError) 422 } 423 val l3HitPPN = l3HitData.ppns 424 val l3HitPerm = l3HitData.perms.getOrElse(0.U.asTypeOf(Vec(PtwL3SectorSize, new PtePermBundle))) 425 val l3HitValid = l3HitData.vs 426 427 // super page 428 val spreplace = ReplacementPolicy.fromString(l2tlbParams.spReplacer, l2tlbParams.spSize) 429 val (spHit, spHitData, spPre, spValid) = { 430 val hitVecT = sp.zipWithIndex.map { case (e, i) => e.hit(vpn_search, io.csr_dup(0).satp.asid, io.csr_dup(0).vsatp.asid, io.csr_dup(0).hgatp.asid, s2xlate = h_search =/= noS2xlate) && spv(i) && (sph(i) === h_search) } 431 val hitVec = hitVecT.map(RegEnable(_, stageReq.fire)) 432 val hitData = ParallelPriorityMux(hitVec zip sp) 433 val hit = ParallelOR(hitVec) 434 435 when (hit && stageDelay_valid_1cycle) { spreplace.access(OHToUInt(hitVec)) } 436 437 spAccessPerf.zip(hitVec).map{ case (s, h) => s := h && stageDelay_valid_1cycle } 438 for (i <- 0 until l2tlbParams.spSize) { 439 XSDebug(stageReq.fire, p"[sp] sp(${i.U}) ${sp(i)} hit:${sp(i).hit(vpn_search, io.csr_dup(0).satp.asid, io.csr_dup(0).vsatp.asid, io.csr_dup(0).hgatp.asid, s2xlate = h_search =/= noS2xlate)} spv:${spv(i)}\n") 440 } 441 XSDebug(stageDelay_valid_1cycle, p"[sp] spHit:${hit} spHitData:${hitData} hitVec:${Binary(VecInit(hitVec).asUInt)}\n") 442 443 VecInit(hitVecT).suggestName(s"sp_hitVecT") 444 VecInit(hitVec).suggestName(s"sp_hitVec") 445 446 (RegEnable(hit, stageDelay(1).fire), 447 RegEnable(hitData, stageDelay(1).fire), 448 RegEnable(hitData.prefetch, stageDelay(1).fire), 449 RegEnable(hitData.v, stageDelay(1).fire)) 450 } 451 val spHitPerm = spHitData.perm.getOrElse(0.U.asTypeOf(new PtePermBundle)) 452 val spHitLevel = spHitData.level.getOrElse(0.U) 453 454 val check_res = Wire(new PageCacheRespBundle) 455 check_res.l1.apply(l1Hit, l1Pre, l1HitPPN) 456 check_res.l2.apply(l2Hit, l2Pre, l2HitPPN, ecc = l2eccError) 457 check_res.l3.apply(l3Hit, l3Pre, l3HitPPN, l3HitPerm, l3eccError, valid = l3HitValid) 458 check_res.sp.apply(spHit, spPre, spHitData.ppn, spHitPerm, false.B, spHitLevel, spValid) 459 460 val resp_res = Reg(new PageCacheRespBundle) 461 when (stageCheck(1).fire) { resp_res := check_res } 462 463 // stageResp bypass 464 val bypassed = Wire(Vec(3, Bool())) 465 bypassed.indices.foreach(i => 466 bypassed(i) := stageResp.bits.bypassed(i) || 467 ValidHoldBypass(refill_bypass(stageResp.bits.req_info.vpn, i, stageResp.bits.req_info.s2xlate), 468 OneCycleValid(stageCheck(1).fire, false.B) || io.refill.valid) 469 ) 470 471 // stageResp bypass to hptw 472 val hptw_bypassed = Wire(Vec(3, Bool())) 473 hptw_bypassed.indices.foreach(i => 474 hptw_bypassed(i) := stageResp.bits.bypassed(i) || 475 ValidHoldBypass(refill_bypass(stageResp.bits.req_info.vpn, i, stageResp.bits.req_info.s2xlate), 476 io.resp.fire) 477 ) 478 479 val isAllStage = stageResp.bits.req_info.s2xlate === allStage 480 val isOnlyStage2 = stageResp.bits.req_info.s2xlate === onlyStage2 481 val stage1Hit = (resp_res.l3.hit || resp_res.sp.hit) && isAllStage 482 val idx = stageResp.bits.req_info.vpn(2, 0) 483 val stage1Pf = !Mux(resp_res.l3.hit, resp_res.l3.v(idx), resp_res.sp.v) 484 io.resp.bits.req_info := stageResp.bits.req_info 485 io.resp.bits.isFirst := stageResp.bits.isFirst 486 io.resp.bits.hit := (resp_res.l3.hit || resp_res.sp.hit) && (!isAllStage || isAllStage && stage1Pf) 487 io.resp.bits.bypassed := (bypassed(2) || (bypassed(1) && !resp_res.l2.hit) || (bypassed(0) && !resp_res.l1.hit)) && !isAllStage 488 io.resp.bits.prefetch := resp_res.l3.pre && resp_res.l3.hit || resp_res.sp.pre && resp_res.sp.hit 489 io.resp.bits.toFsm.l1Hit := resp_res.l1.hit && !stage1Hit && !isOnlyStage2 && !stageResp.bits.isHptwReq 490 io.resp.bits.toFsm.l2Hit := resp_res.l2.hit && !stage1Hit && !isOnlyStage2 && !stageResp.bits.isHptwReq 491 io.resp.bits.toFsm.ppn := Mux(resp_res.l2.hit, resp_res.l2.ppn, resp_res.l1.ppn) 492 io.resp.bits.toFsm.stage1Hit := stage1Hit 493 494 io.resp.bits.isHptwReq := stageResp.bits.isHptwReq 495 io.resp.bits.toHptw.bypassed := (hptw_bypassed(2) || (hptw_bypassed(1) && !resp_res.l2.hit) || (hptw_bypassed(0) && !resp_res.l1.hit)) && stageResp.bits.isHptwReq 496 io.resp.bits.toHptw.id := stageResp.bits.hptwId 497 io.resp.bits.toHptw.l1Hit := resp_res.l1.hit && stageResp.bits.isHptwReq 498 io.resp.bits.toHptw.l2Hit := resp_res.l2.hit && stageResp.bits.isHptwReq 499 io.resp.bits.toHptw.ppn := Mux(resp_res.l2.hit, resp_res.l2.ppn, resp_res.l1.ppn)(ppnLen - 1, 0) 500 io.resp.bits.toHptw.resp.entry.tag := stageResp.bits.req_info.vpn 501 io.resp.bits.toHptw.resp.entry.asid := DontCare 502 io.resp.bits.toHptw.resp.entry.vmid.map(_ := io.csr_dup(0).hgatp.asid) 503 io.resp.bits.toHptw.resp.entry.level.map(_ := Mux(resp_res.l3.hit, 2.U, resp_res.sp.level)) 504 io.resp.bits.toHptw.resp.entry.prefetch := from_pre(stageResp.bits.req_info.source) 505 io.resp.bits.toHptw.resp.entry.ppn := Mux(resp_res.l3.hit, resp_res.l3.ppn(idx), resp_res.sp.ppn)(ppnLen - 1, 0) 506 io.resp.bits.toHptw.resp.entry.perm.map(_ := Mux(resp_res.l3.hit, resp_res.l3.perm(idx), resp_res.sp.perm)) 507 io.resp.bits.toHptw.resp.entry.v := Mux(resp_res.l3.hit, resp_res.l3.v(idx), resp_res.sp.v) 508 io.resp.bits.toHptw.resp.gpf := !io.resp.bits.toHptw.resp.entry.v 509 io.resp.bits.toHptw.resp.gaf := false.B 510 511 io.resp.bits.stage1.entry.map(_.tag := stageResp.bits.req_info.vpn(vpnLen - 1, 3)) 512 io.resp.bits.stage1.entry.map(_.asid := Mux(stageResp.bits.req_info.hasS2xlate(), io.csr_dup(0).vsatp.asid, io.csr_dup(0).satp.asid)) // DontCare 513 io.resp.bits.stage1.entry.map(_.vmid.map(_ := io.csr_dup(0).hgatp.asid)) 514 io.resp.bits.stage1.entry.map(_.level.map(_ := Mux(resp_res.l3.hit, 2.U, Mux(resp_res.sp.hit, resp_res.sp.level, Mux(resp_res.l2.hit, 1.U, 0.U))))) // leaf page is first 515 io.resp.bits.stage1.entry.map(_.prefetch := from_pre(stageResp.bits.req_info.source)) 516 for (i <- 0 until tlbcontiguous) { 517 io.resp.bits.stage1.entry(i).ppn := Mux(resp_res.l3.hit, resp_res.l3.ppn(i)(gvpnLen - 1, sectortlbwidth), Mux(resp_res.sp.hit, resp_res.sp.ppn(gvpnLen - 1, sectortlbwidth), Mux(resp_res.l2.hit, resp_res.l2.ppn(gvpnLen - 1, sectortlbwidth), resp_res.l1.ppn(gvpnLen - 1, sectortlbwidth)))) 518 io.resp.bits.stage1.entry(i).ppn_low := Mux(resp_res.l3.hit, resp_res.l3.ppn(i)(sectortlbwidth - 1, 0), Mux(resp_res.sp.hit, resp_res.sp.ppn(sectortlbwidth - 1, 0), Mux(resp_res.l2.hit, resp_res.l2.ppn(sectortlbwidth - 1, 0), resp_res.l1.ppn(sectortlbwidth - 1, 0)))) 519 io.resp.bits.stage1.entry(i).perm.map(_ := Mux(resp_res.l3.hit, resp_res.l3.perm(i), resp_res.sp.perm)) 520 io.resp.bits.stage1.entry(i).v := Mux(resp_res.l3.hit, resp_res.l3.v(i), Mux(resp_res.sp.hit, resp_res.sp.v, Mux(resp_res.l2.hit, resp_res.l2.v, resp_res.l1.v))) 521 io.resp.bits.stage1.entry(i).pf := !io.resp.bits.stage1.entry(i).v 522 io.resp.bits.stage1.entry(i).af := false.B 523 } 524 io.resp.bits.stage1.pteidx := UIntToOH(idx).asBools 525 io.resp.bits.stage1.not_super := Mux(resp_res.l3.hit, true.B, false.B) 526 io.resp.valid := stageResp.valid 527 XSError(stageResp.valid && resp_res.l3.hit && resp_res.sp.hit, "normal page and super page both hit") 528 XSError(stageResp.valid && io.resp.bits.hit && bypassed(2), "page cache, bypassed but hit") 529 530 // refill Perf 531 val l1RefillPerf = Wire(Vec(l2tlbParams.l1Size, Bool())) 532 val l2RefillPerf = Wire(Vec(l2tlbParams.l2nWays, Bool())) 533 val l3RefillPerf = Wire(Vec(l2tlbParams.l3nWays, Bool())) 534 val spRefillPerf = Wire(Vec(l2tlbParams.spSize, Bool())) 535 l1RefillPerf.map(_ := false.B) 536 l2RefillPerf.map(_ := false.B) 537 l3RefillPerf.map(_ := false.B) 538 spRefillPerf.map(_ := false.B) 539 540 // refill 541 l2.io.w.req <> DontCare 542 l3.io.w.req <> DontCare 543 l2.io.w.req.valid := false.B 544 l3.io.w.req.valid := false.B 545 546 val memRdata = refill.ptes 547 val memPtes = (0 until (l2tlbParams.blockBytes/(XLEN/8))).map(i => memRdata((i+1)*XLEN-1, i*XLEN).asTypeOf(new PteBundle)) 548 val memSelData = io.refill.bits.sel_pte_dup 549 val memPte = memSelData.map(a => a.asTypeOf(new PteBundle)) 550 551 // TODO: handle sfenceLatch outsize 552 when (!flush_dup(0) && refill.levelOH.l1 && !memPte(0).isLeaf() && !memPte(0).isPf(refill.level_dup(0)) && !memPte(0).isAf()) { 553 // val refillIdx = LFSR64()(log2Up(l2tlbParams.l1Size)-1,0) // TODO: may be LRU 554 val refillIdx = replaceWrapper(l1v, ptwl1replace.way) 555 refillIdx.suggestName(s"PtwL1RefillIdx") 556 val rfOH = UIntToOH(refillIdx) 557 l1(refillIdx).refill( 558 refill.req_info_dup(0).vpn, 559 Mux(refill.req_info_dup(0).s2xlate =/= noS2xlate, io.csr_dup(0).vsatp.asid, io.csr_dup(0).satp.asid), 560 io.csr_dup(0).hgatp.asid, 561 memSelData(0), 562 0.U, 563 refill_prefetch_dup(0) 564 ) 565 ptwl1replace.access(refillIdx) 566 l1v := l1v | rfOH 567 l1g := (l1g & ~rfOH) | Mux(memPte(0).perm.g, rfOH, 0.U) 568 l1h(refillIdx) := refill.req_info_dup(0).s2xlate 569 570 for (i <- 0 until l2tlbParams.l1Size) { 571 l1RefillPerf(i) := i.U === refillIdx 572 } 573 574 XSDebug(p"[l1 refill] refillIdx:${refillIdx} refillEntry:${l1(refillIdx).genPtwEntry(refill.req_info_dup(0).vpn, Mux(refill.req_info_dup(0).s2xlate =/= noS2xlate, io.csr_dup(0).vsatp.asid, io.csr_dup(0).satp.asid), memSelData(0), 0.U, prefetch = refill_prefetch_dup(0))}\n") 575 XSDebug(p"[l1 refill] l1v:${Binary(l1v)}->${Binary(l1v | rfOH)} l1g:${Binary(l1g)}->${Binary((l1g & ~rfOH) | Mux(memPte(0).perm.g, rfOH, 0.U))}\n") 576 577 refillIdx.suggestName(s"l1_refillIdx") 578 rfOH.suggestName(s"l1_rfOH") 579 } 580 581 when (!flush_dup(1) && refill.levelOH.l2 && !memPte(1).isLeaf() && !memPte(1).isPf(refill.level_dup(1)) && !memPte(1).isAf()) { 582 val refillIdx = genPtwL2SetIdx(refill.req_info_dup(1).vpn) 583 val victimWay = replaceWrapper(getl2vSet(refill.req_info_dup(1).vpn), ptwl2replace.way(refillIdx)) 584 val victimWayOH = UIntToOH(victimWay) 585 val rfvOH = UIntToOH(Cat(refillIdx, victimWay)) 586 val wdata = Wire(l2EntryType) 587 wdata.gen( 588 vpn = refill.req_info_dup(1).vpn, 589 asid = Mux(refill.req_info_dup(1).s2xlate =/= noS2xlate, io.csr_dup(1).vsatp.asid, io.csr_dup(1).satp.asid), 590 vmid = io.csr_dup(1).hgatp.asid, 591 data = memRdata, 592 levelUInt = 1.U, 593 refill_prefetch_dup(1) 594 ) 595 l2.io.w.apply( 596 valid = true.B, 597 setIdx = refillIdx, 598 data = wdata, 599 waymask = victimWayOH 600 ) 601 ptwl2replace.access(refillIdx, victimWay) 602 l2v := l2v | rfvOH 603 l2g := l2g & ~rfvOH | Mux(Cat(memPtes.map(_.perm.g)).andR, rfvOH, 0.U) 604 l2h(refillIdx)(victimWay) := refill.req_info_dup(1).s2xlate 605 606 for (i <- 0 until l2tlbParams.l2nWays) { 607 l2RefillPerf(i) := i.U === victimWay 608 } 609 610 XSDebug(p"[l2 refill] refillIdx:0x${Hexadecimal(refillIdx)} victimWay:${victimWay} victimWayOH:${Binary(victimWayOH)} rfvOH(in UInt):${Cat(refillIdx, victimWay)}\n") 611 XSDebug(p"[l2 refill] refilldata:0x${wdata}\n") 612 XSDebug(p"[l2 refill] l2v:${Binary(l2v)} -> ${Binary(l2v | rfvOH)}\n") 613 XSDebug(p"[l2 refill] l2g:${Binary(l2g)} -> ${Binary(l2g & ~rfvOH | Mux(Cat(memPtes.map(_.perm.g)).andR, rfvOH, 0.U))}\n") 614 615 refillIdx.suggestName(s"l2_refillIdx") 616 victimWay.suggestName(s"l2_victimWay") 617 victimWayOH.suggestName(s"l2_victimWayOH") 618 rfvOH.suggestName(s"l2_rfvOH") 619 } 620 621 when (!flush_dup(2) && refill.levelOH.l3) { 622 val refillIdx = genPtwL3SetIdx(refill.req_info_dup(2).vpn) 623 val victimWay = replaceWrapper(getl3vSet(refill.req_info_dup(2).vpn), ptwl3replace.way(refillIdx)) 624 val victimWayOH = UIntToOH(victimWay) 625 val rfvOH = UIntToOH(Cat(refillIdx, victimWay)) 626 val wdata = Wire(l3EntryType) 627 wdata.gen( 628 vpn = refill.req_info_dup(2).vpn, 629 asid = Mux(refill.req_info_dup(2).s2xlate =/= noS2xlate, io.csr_dup(2).vsatp.asid, io.csr_dup(2).satp.asid), 630 vmid = io.csr_dup(2).hgatp.asid, 631 data = memRdata, 632 levelUInt = 2.U, 633 refill_prefetch_dup(2) 634 ) 635 l3.io.w.apply( 636 valid = true.B, 637 setIdx = refillIdx, 638 data = wdata, 639 waymask = victimWayOH 640 ) 641 ptwl3replace.access(refillIdx, victimWay) 642 l3v := l3v | rfvOH 643 l3g := l3g & ~rfvOH | Mux(Cat(memPtes.map(_.perm.g)).andR, rfvOH, 0.U) 644 l3h(refillIdx)(victimWay) := refill.req_info_dup(2).s2xlate 645 646 for (i <- 0 until l2tlbParams.l3nWays) { 647 l3RefillPerf(i) := i.U === victimWay 648 } 649 650 XSDebug(p"[l3 refill] refillIdx:0x${Hexadecimal(refillIdx)} victimWay:${victimWay} victimWayOH:${Binary(victimWayOH)} rfvOH(in UInt):${Cat(refillIdx, victimWay)}\n") 651 XSDebug(p"[l3 refill] refilldata:0x${wdata}\n") 652 XSDebug(p"[l3 refill] l3v:${Binary(l3v)} -> ${Binary(l3v | rfvOH)}\n") 653 XSDebug(p"[l3 refill] l3g:${Binary(l3g)} -> ${Binary(l3g & ~rfvOH | Mux(Cat(memPtes.map(_.perm.g)).andR, rfvOH, 0.U))}\n") 654 655 refillIdx.suggestName(s"l3_refillIdx") 656 victimWay.suggestName(s"l3_victimWay") 657 victimWayOH.suggestName(s"l3_victimWayOH") 658 rfvOH.suggestName(s"l3_rfvOH") 659 } 660 661 662 // misc entries: super & invalid 663 when (!flush_dup(0) && refill.levelOH.sp && (memPte(0).isLeaf() || memPte(0).isPf(refill.level_dup(0))) && !memPte(0).isAf()) { 664 val refillIdx = spreplace.way// LFSR64()(log2Up(l2tlbParams.spSize)-1,0) // TODO: may be LRU 665 val rfOH = UIntToOH(refillIdx) 666 sp(refillIdx).refill( 667 refill.req_info_dup(0).vpn, 668 Mux(refill.req_info_dup(0).s2xlate =/= noS2xlate, io.csr_dup(0).vsatp.asid, io.csr_dup(0).satp.asid), 669 io.csr_dup(0).hgatp.asid, 670 memSelData(0), 671 refill.level_dup(2), 672 refill_prefetch_dup(0), 673 !memPte(0).isPf(refill.level_dup(0)), 674 ) 675 spreplace.access(refillIdx) 676 spv := spv | rfOH 677 spg := spg & ~rfOH | Mux(memPte(0).perm.g, rfOH, 0.U) 678 sph(refillIdx) := refill.req_info_dup(0).s2xlate 679 680 for (i <- 0 until l2tlbParams.spSize) { 681 spRefillPerf(i) := i.U === refillIdx 682 } 683 684 XSDebug(p"[sp refill] refillIdx:${refillIdx} refillEntry:${sp(refillIdx).genPtwEntry(refill.req_info_dup(0).vpn, Mux(refill.req_info_dup(0).s2xlate =/= noS2xlate, io.csr_dup(0).vsatp.asid, io.csr_dup(0).satp.asid), memSelData(0), refill.level_dup(0), refill_prefetch_dup(0))}\n") 685 XSDebug(p"[sp refill] spv:${Binary(spv)}->${Binary(spv | rfOH)} spg:${Binary(spg)}->${Binary(spg & ~rfOH | Mux(memPte(0).perm.g, rfOH, 0.U))}\n") 686 687 refillIdx.suggestName(s"sp_refillIdx") 688 rfOH.suggestName(s"sp_rfOH") 689 } 690 691 val l2eccFlush = resp_res.l2.ecc && stageResp_valid_1cycle_dup(0) // RegNext(l2eccError, init = false.B) 692 val l3eccFlush = resp_res.l3.ecc && stageResp_valid_1cycle_dup(1) // RegNext(l3eccError, init = false.B) 693 val eccVpn = stageResp.bits.req_info.vpn 694 695 XSError(l2eccFlush, "l2tlb.cache.l2 ecc error. Should not happen at sim stage") 696 XSError(l3eccFlush, "l2tlb.cache.l3 ecc error. Should not happen at sim stage") 697 when (l2eccFlush) { 698 val flushSetIdxOH = UIntToOH(genPtwL2SetIdx(eccVpn)) 699 val flushMask = VecInit(flushSetIdxOH.asBools.map { a => Fill(l2tlbParams.l2nWays, a.asUInt) }).asUInt 700 l2v := l2v & ~flushMask 701 l2g := l2g & ~flushMask 702 } 703 704 when (l3eccFlush) { 705 val flushSetIdxOH = UIntToOH(genPtwL3SetIdx(eccVpn)) 706 val flushMask = VecInit(flushSetIdxOH.asBools.map { a => Fill(l2tlbParams.l3nWays, a.asUInt) }).asUInt 707 l3v := l3v & ~flushMask 708 l3g := l3g & ~flushMask 709 } 710 711 // sfence for l3 712 val sfence_valid_l3 = sfence_dup(3).valid && !sfence_dup(3).bits.hg && !sfence_dup(3).bits.hv 713 when (sfence_valid_l3) { 714 val l3hhit = VecInit(l3h.flatMap(_.map{a => io.csr_dup(0).priv.virt && a === onlyStage1 || !io.csr_dup(0).priv.virt && a === noS2xlate})).asUInt 715 val sfence_vpn = sfence_dup(3).bits.addr(sfence_dup(3).bits.addr.getWidth-1, offLen) 716 when (sfence_dup(3).bits.rs1/*va*/) { 717 when (sfence_dup(3).bits.rs2) { 718 // all va && all asid 719 l3v := l3v & ~l3hhit 720 } .otherwise { 721 // all va && specific asid except global 722 l3v := l3v & (l3g | ~l3hhit) 723 } 724 } .otherwise { 725 // val flushMask = UIntToOH(genTlbL2Idx(sfence.bits.addr(sfence.bits.addr.getWidth-1, offLen))) 726 val flushSetIdxOH = UIntToOH(genPtwL3SetIdx(sfence_vpn)) 727 // val flushMask = VecInit(flushSetIdxOH.asBools.map(Fill(l2tlbParams.l3nWays, _.asUInt))).asUInt 728 val flushMask = VecInit(flushSetIdxOH.asBools.map { a => Fill(l2tlbParams.l3nWays, a.asUInt) }).asUInt 729 flushSetIdxOH.suggestName(s"sfence_nrs1_flushSetIdxOH") 730 flushMask.suggestName(s"sfence_nrs1_flushMask") 731 732 when (sfence_dup(3).bits.rs2) { 733 // specific leaf of addr && all asid 734 l3v := l3v & ~flushMask & ~l3hhit 735 } .otherwise { 736 // specific leaf of addr && specific asid 737 l3v := l3v & (~flushMask | l3g | ~l3hhit) 738 } 739 } 740 } 741 742 // hfencev, simple implementation for l3 743 val hfencev_valid_l3 = sfence_dup(3).valid && sfence_dup(3).bits.hv 744 when(hfencev_valid_l3) { 745 val flushMask = VecInit(l3h.flatMap(_.map(_ === onlyStage1))).asUInt 746 l3v := l3v & ~flushMask // all VS-stage l3 pte 747 } 748 749 // hfenceg, simple implementation for l3 750 val hfenceg_valid_l3 = sfence_dup(3).valid && sfence_dup(3).bits.hg 751 when(hfenceg_valid_l3) { 752 val flushMask = VecInit(l3h.flatMap(_.map(_ === onlyStage2))).asUInt 753 l3v := l3v & ~flushMask // all G-stage l3 pte 754 } 755 756 757 val l1asidhit = VecInit(l1asids.map(_ === sfence_dup(0).bits.id)).asUInt 758 val spasidhit = VecInit(spasids.map(_ === sfence_dup(0).bits.id)).asUInt 759 val sfence_valid = sfence_dup(0).valid && !sfence_dup(0).bits.hg && !sfence_dup(0).bits.hv 760 when (sfence_valid) { 761 val l1vmidhit = VecInit(l1vmids.map(_.getOrElse(0.U) === io.csr_dup(0).hgatp.asid)).asUInt 762 val spvmidhit = VecInit(spvmids.map(_.getOrElse(0.U) === io.csr_dup(0).hgatp.asid)).asUInt 763 val l1hhit = VecInit(l1h.map{a => io.csr_dup(0).priv.virt && a === onlyStage1 || !io.csr_dup(0).priv.virt && a === noS2xlate}).asUInt 764 val sphhit = VecInit(sph.map{a => io.csr_dup(0).priv.virt && a === onlyStage1 || !io.csr_dup(0).priv.virt && a === noS2xlate}).asUInt 765 val l2hhit = VecInit(l2h.flatMap(_.map{a => io.csr_dup(0).priv.virt && a === onlyStage1 || !io.csr_dup(0).priv.virt && a === noS2xlate})).asUInt 766 val sfence_vpn = sfence_dup(0).bits.addr(sfence_dup(0).bits.addr.getWidth-1, offLen) 767 768 when (sfence_dup(0).bits.rs1/*va*/) { 769 when (sfence_dup(0).bits.rs2) { 770 // all va && all asid 771 l2v := l2v & ~l2hhit 772 l1v := l1v & ~(l1hhit & VecInit(l1vmidhit.asBools.map{a => io.csr_dup(0).priv.virt && a || !io.csr_dup(0).priv.virt}).asUInt) 773 spv := spv & ~(sphhit & VecInit(spvmidhit.asBools.map{a => io.csr_dup(0).priv.virt && a || !io.csr_dup(0).priv.virt}).asUInt) 774 } .otherwise { 775 // all va && specific asid except global 776 l2v := l2v & (l2g | ~l2hhit) 777 l1v := l1v & ~(~l1g & l1hhit & l1asidhit & VecInit(l1vmidhit.asBools.map{a => io.csr_dup(0).priv.virt && a || !io.csr_dup(0).priv.virt}).asUInt) 778 spv := spv & ~(~spg & sphhit & spasidhit & VecInit(spvmidhit.asBools.map{a => io.csr_dup(0).priv.virt && a || !io.csr_dup(0).priv.virt}).asUInt) 779 } 780 } .otherwise { 781 when (sfence_dup(0).bits.rs2) { 782 // specific leaf of addr && all asid 783 spv := spv & ~(sphhit & VecInit(sp.map(_.hit(sfence_vpn, sfence_dup(0).bits.id, sfence_dup(0).bits.id, io.csr_dup(0).hgatp.asid, ignoreAsid = true, s2xlate = io.csr_dup(0).priv.virt))).asUInt) 784 } .otherwise { 785 // specific leaf of addr && specific asid 786 spv := spv & ~(~spg & sphhit & VecInit(sp.map(_.hit(sfence_vpn, sfence_dup(0).bits.id, sfence_dup(0).bits.id, io.csr_dup(0).hgatp.asid, s2xlate = io.csr_dup(0).priv.virt))).asUInt) 787 } 788 } 789 } 790 791 val hfencev_valid = sfence_dup(0).valid && sfence_dup(0).bits.hv 792 when (hfencev_valid) { 793 val l1vmidhit = VecInit(l1vmids.map(_.getOrElse(0.U) === io.csr_dup(0).hgatp.asid)).asUInt 794 val spvmidhit = VecInit(spvmids.map(_.getOrElse(0.U) === io.csr_dup(0).hgatp.asid)).asUInt 795 val l1hhit = VecInit(l1h.map(_ === onlyStage1)).asUInt 796 val sphhit = VecInit(sph.map(_ === onlyStage1)).asUInt 797 val l2hhit = VecInit(l2h.flatMap(_.map(_ === onlyStage1))).asUInt 798 val hfencev_vpn = sfence_dup(0).bits.addr(sfence_dup(0).bits.addr.getWidth-1, offLen) 799 when(sfence_dup(0).bits.rs1) { 800 when(sfence_dup(0).bits.rs2) { 801 l2v := l2v & ~l2hhit 802 l1v := l1v & ~(l1hhit & l1vmidhit) 803 spv := spv & ~(sphhit & spvmidhit) 804 }.otherwise { 805 l2v := l2v & (l2g | ~l2hhit) 806 l1v := l1v & ~(~l1g & l1hhit & l1asidhit & l1vmidhit) 807 spv := spv & ~(~spg & sphhit & spasidhit & spvmidhit) 808 } 809 }.otherwise { 810 when(sfence_dup(0).bits.rs2) { 811 spv := spv & ~(sphhit & VecInit(sp.map(_.hit(hfencev_vpn, sfence_dup(0).bits.id, sfence_dup(0).bits.id, io.csr_dup(0).hgatp.asid, ignoreAsid = true, s2xlate = true.B))).asUInt) 812 }.otherwise { 813 spv := spv & ~(~spg & sphhit & VecInit(sp.map(_.hit(hfencev_vpn, sfence_dup(0).bits.id, sfence_dup(0).bits.id, io.csr_dup(0).hgatp.asid, s2xlate = true.B))).asUInt) 814 } 815 } 816 } 817 818 819 val hfenceg_valid = sfence_dup(0).valid && sfence_dup(0).bits.hg 820 when(hfenceg_valid) { 821 val l1vmidhit = VecInit(l1vmids.map(_.getOrElse(0.U) === sfence_dup(0).bits.id)).asUInt 822 val spvmidhit = VecInit(spvmids.map(_.getOrElse(0.U) === sfence_dup(0).bits.id)).asUInt 823 val l1hhit = VecInit(l1h.map(_ === onlyStage2)).asUInt 824 val sphhit = VecInit(sph.map(_ === onlyStage2)).asUInt 825 val l2hhit = VecInit(l2h.flatMap(_.map(_ === onlyStage2))).asUInt 826 val hfenceg_gvpn = (sfence_dup(0).bits.addr << 2)(sfence_dup(0).bits.addr.getWidth - 1, offLen) 827 when(sfence_dup(0).bits.rs1) { 828 when(sfence_dup(0).bits.rs2) { 829 l2v := l2v & ~l2hhit 830 l1v := l1v & ~l1hhit 831 spv := spv & ~sphhit 832 }.otherwise { 833 l2v := l2v & ~l2hhit 834 l1v := l1v & ~(l1hhit & l1vmidhit) 835 spv := spv & ~(sphhit & spvmidhit) 836 } 837 }.otherwise { 838 when(sfence_dup(0).bits.rs2) { 839 spv := spv & ~(sphhit & VecInit(sp.map(_.hit(hfenceg_gvpn, 0.U, 0.U, sfence_dup(0).bits.id, ignoreAsid = true, s2xlate = false.B))).asUInt) 840 }.otherwise { 841 spv := spv & ~(~spg & sphhit & VecInit(sp.map(_.hit(hfenceg_gvpn, 0.U, 0.U, sfence_dup(0).bits.id, ignoreAsid = true, s2xlate = true.B))).asUInt) 842 } 843 } 844 } 845 846 def InsideStageConnect(in: DecoupledIO[PtwCacheReq], out: DecoupledIO[PtwCacheReq], inFire: Bool): Unit = { 847 in.ready := !in.valid || out.ready 848 out.valid := in.valid 849 out.bits := in.bits 850 out.bits.bypassed.zip(in.bits.bypassed).zipWithIndex.map{ case (b, i) => 851 val bypassed_reg = Reg(Bool()) 852 val bypassed_wire = refill_bypass(in.bits.req_info.vpn, i, in.bits.req_info.s2xlate) && io.refill.valid 853 when (inFire) { bypassed_reg := bypassed_wire } 854 .elsewhen (io.refill.valid) { bypassed_reg := bypassed_reg || bypassed_wire } 855 856 b._1 := b._2 || (bypassed_wire || (bypassed_reg && !inFire)) 857 } 858 } 859 860 // Perf Count 861 val resp_l3 = resp_res.l3.hit 862 val resp_sp = resp_res.sp.hit 863 val resp_l1_pre = resp_res.l1.pre 864 val resp_l2_pre = resp_res.l2.pre 865 val resp_l3_pre = resp_res.l3.pre 866 val resp_sp_pre = resp_res.sp.pre 867 val base_valid_access_0 = !from_pre(io.resp.bits.req_info.source) && io.resp.fire 868 XSPerfAccumulate("access", base_valid_access_0) 869 XSPerfAccumulate("l1_hit", base_valid_access_0 && io.resp.bits.toFsm.l1Hit && !io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 870 XSPerfAccumulate("l2_hit", base_valid_access_0 && io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 871 XSPerfAccumulate("l3_hit", base_valid_access_0 && resp_l3) 872 XSPerfAccumulate("sp_hit", base_valid_access_0 && resp_sp) 873 XSPerfAccumulate("pte_hit",base_valid_access_0 && io.resp.bits.hit) 874 875 XSPerfAccumulate("l1_hit_pre", base_valid_access_0 && resp_l1_pre && io.resp.bits.toFsm.l1Hit && !io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 876 XSPerfAccumulate("l2_hit_pre", base_valid_access_0 && resp_l2_pre && io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 877 XSPerfAccumulate("l3_hit_pre", base_valid_access_0 && resp_l3_pre && resp_l3) 878 XSPerfAccumulate("sp_hit_pre", base_valid_access_0 && resp_sp_pre && resp_sp) 879 XSPerfAccumulate("pte_hit_pre",base_valid_access_0 && (resp_l3_pre && resp_l3 || resp_sp_pre && resp_sp) && io.resp.bits.hit) 880 881 val base_valid_access_1 = from_pre(io.resp.bits.req_info.source) && io.resp.fire 882 XSPerfAccumulate("pre_access", base_valid_access_1) 883 XSPerfAccumulate("pre_l1_hit", base_valid_access_1 && io.resp.bits.toFsm.l1Hit && !io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 884 XSPerfAccumulate("pre_l2_hit", base_valid_access_1 && io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 885 XSPerfAccumulate("pre_l3_hit", base_valid_access_1 && resp_l3) 886 XSPerfAccumulate("pre_sp_hit", base_valid_access_1 && resp_sp) 887 XSPerfAccumulate("pre_pte_hit",base_valid_access_1 && io.resp.bits.hit) 888 889 XSPerfAccumulate("pre_l1_hit_pre", base_valid_access_1 && resp_l1_pre && io.resp.bits.toFsm.l1Hit && !io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 890 XSPerfAccumulate("pre_l2_hit_pre", base_valid_access_1 && resp_l2_pre && io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 891 XSPerfAccumulate("pre_l3_hit_pre", base_valid_access_1 && resp_l3_pre && resp_l3) 892 XSPerfAccumulate("pre_sp_hit_pre", base_valid_access_1 && resp_sp_pre && resp_sp) 893 XSPerfAccumulate("pre_pte_hit_pre",base_valid_access_1 && (resp_l3_pre && resp_l3 || resp_sp_pre && resp_sp) && io.resp.bits.hit) 894 895 val base_valid_access_2 = stageResp.bits.isFirst && !from_pre(io.resp.bits.req_info.source) && io.resp.fire 896 XSPerfAccumulate("access_first", base_valid_access_2) 897 XSPerfAccumulate("l1_hit_first", base_valid_access_2 && io.resp.bits.toFsm.l1Hit && !io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 898 XSPerfAccumulate("l2_hit_first", base_valid_access_2 && io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 899 XSPerfAccumulate("l3_hit_first", base_valid_access_2 && resp_l3) 900 XSPerfAccumulate("sp_hit_first", base_valid_access_2 && resp_sp) 901 XSPerfAccumulate("pte_hit_first",base_valid_access_2 && io.resp.bits.hit) 902 903 XSPerfAccumulate("l1_hit_pre_first", base_valid_access_2 && resp_l1_pre && io.resp.bits.toFsm.l1Hit && !io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 904 XSPerfAccumulate("l2_hit_pre_first", base_valid_access_2 && resp_l2_pre && io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 905 XSPerfAccumulate("l3_hit_pre_first", base_valid_access_2 && resp_l3_pre && resp_l3) 906 XSPerfAccumulate("sp_hit_pre_first", base_valid_access_2 && resp_sp_pre && resp_sp) 907 XSPerfAccumulate("pte_hit_pre_first",base_valid_access_2 && (resp_l3_pre && resp_l3 || resp_sp_pre && resp_sp) && io.resp.bits.hit) 908 909 val base_valid_access_3 = stageResp.bits.isFirst && from_pre(io.resp.bits.req_info.source) && io.resp.fire 910 XSPerfAccumulate("pre_access_first", base_valid_access_3) 911 XSPerfAccumulate("pre_l1_hit_first", base_valid_access_3 && io.resp.bits.toFsm.l1Hit && !io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 912 XSPerfAccumulate("pre_l2_hit_first", base_valid_access_3 && io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 913 XSPerfAccumulate("pre_l3_hit_first", base_valid_access_3 && resp_l3) 914 XSPerfAccumulate("pre_sp_hit_first", base_valid_access_3 && resp_sp) 915 XSPerfAccumulate("pre_pte_hit_first", base_valid_access_3 && io.resp.bits.hit) 916 917 XSPerfAccumulate("pre_l1_hit_pre_first", base_valid_access_3 && resp_l1_pre && io.resp.bits.toFsm.l1Hit && !io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 918 XSPerfAccumulate("pre_l2_hit_pre_first", base_valid_access_3 && resp_l2_pre && io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 919 XSPerfAccumulate("pre_l3_hit_pre_first", base_valid_access_3 && resp_l3_pre && resp_l3) 920 XSPerfAccumulate("pre_sp_hit_pre_first", base_valid_access_3 && resp_sp_pre && resp_sp) 921 XSPerfAccumulate("pre_pte_hit_pre_first",base_valid_access_3 && (resp_l3_pre && resp_l3 || resp_sp_pre && resp_sp) && io.resp.bits.hit) 922 923 XSPerfAccumulate("rwHarzad", io.req.valid && !io.req.ready) 924 XSPerfAccumulate("out_blocked", io.resp.valid && !io.resp.ready) 925 l1AccessPerf.zipWithIndex.map{ case (l, i) => XSPerfAccumulate(s"L1AccessIndex${i}", l) } 926 l2AccessPerf.zipWithIndex.map{ case (l, i) => XSPerfAccumulate(s"L2AccessIndex${i}", l) } 927 l3AccessPerf.zipWithIndex.map{ case (l, i) => XSPerfAccumulate(s"L3AccessIndex${i}", l) } 928 spAccessPerf.zipWithIndex.map{ case (l, i) => XSPerfAccumulate(s"SPAccessIndex${i}", l) } 929 l1RefillPerf.zipWithIndex.map{ case (l, i) => XSPerfAccumulate(s"L1RefillIndex${i}", l) } 930 l2RefillPerf.zipWithIndex.map{ case (l, i) => XSPerfAccumulate(s"L2RefillIndex${i}", l) } 931 l3RefillPerf.zipWithIndex.map{ case (l, i) => XSPerfAccumulate(s"L3RefillIndex${i}", l) } 932 spRefillPerf.zipWithIndex.map{ case (l, i) => XSPerfAccumulate(s"SPRefillIndex${i}", l) } 933 934 XSPerfAccumulate("l1Refill", Cat(l1RefillPerf).orR) 935 XSPerfAccumulate("l2Refill", Cat(l2RefillPerf).orR) 936 XSPerfAccumulate("l3Refill", Cat(l3RefillPerf).orR) 937 XSPerfAccumulate("spRefill", Cat(spRefillPerf).orR) 938 XSPerfAccumulate("l1Refill_pre", Cat(l1RefillPerf).orR && refill_prefetch_dup(0)) 939 XSPerfAccumulate("l2Refill_pre", Cat(l2RefillPerf).orR && refill_prefetch_dup(0)) 940 XSPerfAccumulate("l3Refill_pre", Cat(l3RefillPerf).orR && refill_prefetch_dup(0)) 941 XSPerfAccumulate("spRefill_pre", Cat(spRefillPerf).orR && refill_prefetch_dup(0)) 942 943 // debug 944 XSDebug(sfence_dup(0).valid, p"[sfence] original v and g vector:\n") 945 XSDebug(sfence_dup(0).valid, p"[sfence] l1v:${Binary(l1v)}\n") 946 XSDebug(sfence_dup(0).valid, p"[sfence] l2v:${Binary(l2v)}\n") 947 XSDebug(sfence_dup(0).valid, p"[sfence] l3v:${Binary(l3v)}\n") 948 XSDebug(sfence_dup(0).valid, p"[sfence] l3g:${Binary(l3g)}\n") 949 XSDebug(sfence_dup(0).valid, p"[sfence] spv:${Binary(spv)}\n") 950 XSDebug(RegNext(sfence_dup(0).valid), p"[sfence] new v and g vector:\n") 951 XSDebug(RegNext(sfence_dup(0).valid), p"[sfence] l1v:${Binary(l1v)}\n") 952 XSDebug(RegNext(sfence_dup(0).valid), p"[sfence] l2v:${Binary(l2v)}\n") 953 XSDebug(RegNext(sfence_dup(0).valid), p"[sfence] l3v:${Binary(l3v)}\n") 954 XSDebug(RegNext(sfence_dup(0).valid), p"[sfence] l3g:${Binary(l3g)}\n") 955 XSDebug(RegNext(sfence_dup(0).valid), p"[sfence] spv:${Binary(spv)}\n") 956 957 val perfEvents = Seq( 958 ("access ", base_valid_access_0 ), 959 ("l1_hit ", l1Hit ), 960 ("l2_hit ", l2Hit ), 961 ("l3_hit ", l3Hit ), 962 ("sp_hit ", spHit ), 963 ("pte_hit ", l3Hit || spHit ), 964 ("rwHarzad ", io.req.valid && !io.req.ready ), 965 ("out_blocked ", io.resp.valid && !io.resp.ready), 966 ) 967 generatePerfEvent() 968} 969