1/*************************************************************************************** 2* Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences 3* Copyright (c) 2020-2021 Peng Cheng Laboratory 4* 5* XiangShan is licensed under Mulan PSL v2. 6* You can use this software according to the terms and conditions of the Mulan PSL v2. 7* You may obtain a copy of Mulan PSL v2 at: 8* http://license.coscl.org.cn/MulanPSL2 9* 10* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, 11* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, 12* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. 13* 14* See the Mulan PSL v2 for more details. 15***************************************************************************************/ 16 17package xiangshan.cache.mmu 18 19import chipsalliance.rocketchip.config.Parameters 20import chisel3._ 21import chisel3.util._ 22import chisel3.internal.naming.chiselName 23import xiangshan._ 24import xiangshan.cache.{HasDCacheParameters, MemoryOpConstants} 25import utils._ 26import freechips.rocketchip.diplomacy.{LazyModule, LazyModuleImp} 27import freechips.rocketchip.tilelink._ 28 29/* ptw cache caches the page table of all the three layers 30 * ptw cache resp at next cycle 31 * the cache should not be blocked 32 * when miss queue if full, just block req outside 33 */ 34 35class PageCachePerPespBundle(implicit p: Parameters) extends PtwBundle { 36 val hit = Bool() 37 val pre = Bool() 38 val ppn = UInt(ppnLen.W) 39 val perm = new PtePermBundle() 40 val ecc = Bool() 41 val level = UInt(2.W) 42 43 def apply(hit: Bool, pre: Bool, ppn: UInt, perm: PtePermBundle = 0.U.asTypeOf(new PtePermBundle()), 44 ecc: Bool = false.B, level: UInt = 0.U) { 45 this.hit := hit && !ecc 46 this.pre := pre 47 this.ppn := ppn 48 this.perm := perm 49 this.ecc := ecc && hit 50 this.level := level 51 } 52} 53 54class PageCacheRespBundle(implicit p: Parameters) extends PtwBundle { 55 val l1 = new PageCachePerPespBundle 56 val l2 = new PageCachePerPespBundle 57 val l3 = new PageCachePerPespBundle 58 val sp = new PageCachePerPespBundle 59} 60 61class PtwCacheReq(implicit p: Parameters) extends PtwBundle { 62 val req_info = new L2TlbInnerBundle() 63 val isFirst = Bool() 64} 65 66class PtwCacheIO()(implicit p: Parameters) extends MMUIOBaseBundle with HasPtwConst { 67 val req = Flipped(DecoupledIO(new PtwCacheReq())) 68 val resp = DecoupledIO(new Bundle { 69 val req_info = new L2TlbInnerBundle() 70 val hit = Bool() 71 val prefetch = Bool() // is the entry fetched by prefetch 72 val toFsm = new Bundle { 73 val l1Hit = Bool() 74 val l2Hit = Bool() 75 val ppn = UInt(ppnLen.W) 76 } 77 val toTlb = new PtwEntry(tagLen = vpnLen, hasPerm = true, hasLevel = true) 78 }) 79 val refill = Flipped(ValidIO(new Bundle { 80 val ptes = UInt(blockBits.W) 81 val req_info = new L2TlbInnerBundle() 82 val level = UInt(log2Up(Level).W) 83 val addr_low = UInt((log2Up(l2tlbParams.blockBytes) - log2Up(XLEN/8)).W) 84 })) 85} 86 87@chiselName 88class PtwCache()(implicit p: Parameters) extends XSModule with HasPtwConst { 89 val io = IO(new PtwCacheIO) 90 91 val ecc = Code.fromString(l2tlbParams.ecc) 92 val l2EntryType = new PTWEntriesWithEcc(ecc, num = PtwL2SectorSize, tagLen = PtwL2TagLen, level = 1, hasPerm = false) 93 val l3EntryType = new PTWEntriesWithEcc(ecc, num = PtwL3SectorSize, tagLen = PtwL3TagLen, level = 2, hasPerm = true) 94 95 // TODO: four caches make the codes dirty, think about how to deal with it 96 97 val sfence = io.sfence 98 val refill = io.refill.bits 99 val refill_prefetch = from_pre(io.refill.bits.req_info.source) 100 val flush = sfence.valid || io.csr.satp.changed 101 102 // when refill, refuce to accept new req 103 val rwHarzad = if (sramSinglePort) io.refill.valid else false.B 104 105 // handle hand signal and req_info 106 val stage1 = Wire(Decoupled(new PtwCacheReq())) // enq stage & read page cache valid 107 val stage2 = Wire(Vec(2, Decoupled(new PtwCacheReq()))) // page cache resp & check hit & check ecc 108 val stage3 = Wire(Decoupled(new PtwCacheReq())) // deq stage 109 /* stage1.valid && stage2(0).ready : stage1 (in) -> stage2 110 * stage2(1).valid && stage3.ready : stage2 -> stage3 111 * stage3.valid && io.resp.ready : stage3 (out) -> outside 112 */ 113 stage1 <> io.req 114 PipelineConnect(stage1, stage2(0), stage2(1).ready, flush, rwHarzad) 115 InsideStageConnect(stage2(0), stage2(1)) 116 PipelineConnect(stage2(1), stage3, io.resp.ready, flush) 117 stage3.ready := !stage3.valid || io.resp.ready 118 119 // l1: level 0 non-leaf pte 120 val l1 = Reg(Vec(l2tlbParams.l1Size, new PtwEntry(tagLen = PtwL1TagLen))) 121 val l1v = RegInit(0.U(l2tlbParams.l1Size.W)) 122 val l1g = Reg(UInt(l2tlbParams.l1Size.W)) 123 val l1asids = Reg(Vec(l2tlbParams.l1Size, UInt(AsidLength.W))) 124 125 // l2: level 1 non-leaf pte 126 val l2 = Module(new SRAMTemplate( 127 l2EntryType, 128 set = l2tlbParams.l2nSets, 129 way = l2tlbParams.l2nWays, 130 singlePort = sramSinglePort 131 )) 132 val l2v = RegInit(0.U((l2tlbParams.l2nSets * l2tlbParams.l2nWays).W)) 133 val l2g = Reg(UInt((l2tlbParams.l2nSets * l2tlbParams.l2nWays).W)) 134 val l2asids = Reg(Vec(l2tlbParams.l2nSets, Vec(l2tlbParams.l2nWays, UInt(AsidLength.W)))) 135 def getl2vSet(vpn: UInt) = { 136 require(log2Up(l2tlbParams.l2nWays) == log2Down(l2tlbParams.l2nWays)) 137 val set = genPtwL2SetIdx(vpn) 138 require(set.getWidth == log2Up(l2tlbParams.l2nSets)) 139 val l2vVec = l2v.asTypeOf(Vec(l2tlbParams.l2nSets, UInt(l2tlbParams.l2nWays.W))) 140 l2vVec(set) 141 } 142 def getl2asidSet(vpn: UInt) = { 143 require(log2Up(l2tlbParams.l2nWays) == log2Down(l2tlbParams.l2nWays)) 144 val set = genPtwL2SetIdx(vpn) 145 require(set.getWidth == log2Up(l2tlbParams.l2nSets)) 146 l2asids(set) 147 } 148 149 // l3: level 2 leaf pte of 4KB pages 150 val l3 = Module(new SRAMTemplate( 151 l3EntryType, 152 set = l2tlbParams.l3nSets, 153 way = l2tlbParams.l3nWays, 154 singlePort = sramSinglePort 155 )) 156 val l3v = RegInit(0.U((l2tlbParams.l3nSets * l2tlbParams.l3nWays).W)) 157 val l3g = Reg(UInt((l2tlbParams.l3nSets * l2tlbParams.l3nWays).W)) 158 val l3asids = Reg(Vec(l2tlbParams.l3nSets, Vec(l2tlbParams.l3nWays, UInt(AsidLength.W)))) 159 def getl3vSet(vpn: UInt) = { 160 require(log2Up(l2tlbParams.l3nWays) == log2Down(l2tlbParams.l3nWays)) 161 val set = genPtwL3SetIdx(vpn) 162 require(set.getWidth == log2Up(l2tlbParams.l3nSets)) 163 val l3vVec = l3v.asTypeOf(Vec(l2tlbParams.l3nSets, UInt(l2tlbParams.l3nWays.W))) 164 l3vVec(set) 165 } 166 def getl3asidSet(vpn: UInt) = { 167 require(log2Up(l2tlbParams.l3nWays) == log2Down(l2tlbParams.l3nWays)) 168 val set = genPtwL3SetIdx(vpn) 169 require(set.getWidth == log2Up(l2tlbParams.l3nSets)) 170 l3asids(set) 171 } 172 173 // sp: level 0/1 leaf pte of 1GB/2MB super pages 174 val sp = Reg(Vec(l2tlbParams.spSize, new PtwEntry(tagLen = SPTagLen, hasPerm = true, hasLevel = true))) 175 val spv = RegInit(0.U(l2tlbParams.spSize.W)) 176 val spg = Reg(UInt(l2tlbParams.spSize.W)) 177 val spasids = Reg(Vec(l2tlbParams.spSize, UInt(AsidLength.W))) 178 179 // Access Perf 180 val l1AccessPerf = Wire(Vec(l2tlbParams.l1Size, Bool())) 181 val l2AccessPerf = Wire(Vec(l2tlbParams.l2nWays, Bool())) 182 val l3AccessPerf = Wire(Vec(l2tlbParams.l3nWays, Bool())) 183 val spAccessPerf = Wire(Vec(l2tlbParams.spSize, Bool())) 184 l1AccessPerf.map(_ := false.B) 185 l2AccessPerf.map(_ := false.B) 186 l3AccessPerf.map(_ := false.B) 187 spAccessPerf.map(_ := false.B) 188 189 // stage1 & stage2, read page cache and data resp 190 191 val cache_read_valid = OneCycleValid(stage1.fire, flush) 192 // l1 193 val ptwl1replace = ReplacementPolicy.fromString(l2tlbParams.l1Replacer, l2tlbParams.l1Size) 194 val (l1Hit, l1HitPPN, l1Pre) = { 195 val hitVecT = l1.zipWithIndex.map { case (e, i) => e.hit(stage1.bits.req_info.vpn, io.csr.satp.asid) && l1v(i) } 196 val hitVec = hitVecT.map(RegEnable(_, stage1.fire)) 197 val hitPPN = ParallelPriorityMux(hitVec zip l1.map(_.ppn)) 198 val hitPre = ParallelPriorityMux(hitVec zip l1.map(_.prefetch)) 199 val hit = ParallelOR(hitVec) && cache_read_valid 200 201 when (hit) { ptwl1replace.access(OHToUInt(hitVec)) } 202 203 l1AccessPerf.zip(hitVec).map{ case (l, h) => l := h && RegNext(stage1.fire)} 204 for (i <- 0 until l2tlbParams.l1Size) { 205 XSDebug(stage1.fire, p"[l1] l1(${i.U}) ${l1(i)} hit:${l1(i).hit(stage1.bits.req_info.vpn, io.csr.satp.asid)}\n") 206 } 207 XSDebug(stage1.fire, p"[l1] l1v:${Binary(l1v)} hitVecT:${Binary(VecInit(hitVecT).asUInt)}\n") 208 XSDebug(stage2(0).valid, p"[l1] l1Hit:${hit} l1HitPPN:0x${Hexadecimal(hitPPN)} hitVec:${VecInit(hitVec).asUInt}\n") 209 210 VecInit(hitVecT).suggestName(s"l1_hitVecT") 211 VecInit(hitVec).suggestName(s"l1_hitVec") 212 213 (hit, hitPPN, hitPre) 214 } 215 216 // l2 217 val ptwl2replace = ReplacementPolicy.fromString(l2tlbParams.l2Replacer,l2tlbParams.l2nWays,l2tlbParams.l2nSets) 218 val (l2Hit, l2HitPPN, l2Pre, l2eccError) = { 219 val ridx = genPtwL2SetIdx(stage1.bits.req_info.vpn) 220 val vidx = RegEnable(VecInit(getl2vSet(stage1.bits.req_info.vpn).asBools), stage1.fire) 221 val asids_idx = RegEnable(getl2asidSet(stage1.bits.req_info.vpn), stage1.fire) 222 l2.io.r.req.valid := stage1.fire 223 l2.io.r.req.bits.apply(setIdx = ridx) 224 val ramDatas = l2.io.r.resp.data 225 val hitVec = VecInit(ramDatas.zip(vidx).map { case (wayData, v) => wayData.entries.hit(stage2(0).bits.req_info.vpn, io.csr.satp.asid) && v }) 226 val hitWayEntry = ParallelPriorityMux(hitVec zip ramDatas) 227 val hitWayData = hitWayEntry.entries 228 val hit = ParallelOR(hitVec) && cache_read_valid && RegNext(l2.io.r.req.ready, init = false.B) 229 val hitWay = ParallelPriorityMux(hitVec zip (0 until l2tlbParams.l2nWays).map(_.U)) 230 val eccError = hitWayEntry.decode() 231 232 ridx.suggestName(s"l2_ridx") 233 vidx.suggestName(s"l2_vidx") 234 ramDatas.suggestName(s"l2_ramDatas") 235 hitVec.suggestName(s"l2_hitVec") 236 hitWayData.suggestName(s"l2_hitWayData") 237 hitWay.suggestName(s"l2_hitWay") 238 239 when (hit) { ptwl2replace.access(genPtwL2SetIdx(stage2(0).bits.req_info.vpn), hitWay) } 240 241 l2AccessPerf.zip(hitVec).map{ case (l, h) => l := h && RegNext(stage1.fire) } 242 XSDebug(stage1.fire, p"[l2] ridx:0x${Hexadecimal(ridx)}\n") 243 for (i <- 0 until l2tlbParams.l2nWays) { 244 XSDebug(RegNext(stage1.fire), p"[l2] ramDatas(${i.U}) ${ramDatas(i)} l2v:${vidx(i)} hit:${ramDatas(i).entries.hit(stage2(0).bits.req_info.vpn, io.csr.satp.asid)}\n") 245 } 246 XSDebug(stage2(0).valid, p"[l2] l2Hit:${hit} l2HitPPN:0x${Hexadecimal(hitWayData.ppns(genPtwL2SectorIdx(stage2(0).bits.req_info.vpn)))} hitVec:${Binary(hitVec.asUInt)} hitWay:${hitWay} vidx:${Binary(vidx.asUInt)}\n") 247 248 (hit, hitWayData.ppns(genPtwL2SectorIdx(stage2(0).bits.req_info.vpn)), hitWayData.prefetch, eccError) 249 } 250 251 // l3 252 val ptwl3replace = ReplacementPolicy.fromString(l2tlbParams.l3Replacer,l2tlbParams.l3nWays,l2tlbParams.l3nSets) 253 val (l3Hit, l3HitData, l3Pre, l3eccError) = { 254 val ridx = genPtwL3SetIdx(stage1.bits.req_info.vpn) 255 val vidx = RegEnable(VecInit(getl3vSet(stage1.bits.req_info.vpn).asBools), stage1.fire) 256 val asids_idx = RegEnable(getl3asidSet(stage1.bits.req_info.vpn), stage1.fire) 257 l3.io.r.req.valid := stage1.fire 258 l3.io.r.req.bits.apply(setIdx = ridx) 259 val ramDatas = l3.io.r.resp.data 260 val hitVec = VecInit(ramDatas.zip(vidx).map{ case (wayData, v) => wayData.entries.hit(stage2(0).bits.req_info.vpn, io.csr.satp.asid) && v }) 261 val hitWayEntry = ParallelPriorityMux(hitVec zip ramDatas) 262 val hitWayData = hitWayEntry.entries 263 val hitWayEcc = hitWayEntry.ecc 264 val hit = ParallelOR(hitVec) && cache_read_valid && RegNext(l3.io.r.req.ready, init = false.B) 265 val hitWay = ParallelPriorityMux(hitVec zip (0 until l2tlbParams.l3nWays).map(_.U)) 266 val eccError = hitWayEntry.decode() 267 268 when (hit) { ptwl3replace.access(genPtwL3SetIdx(stage2(0).bits.req_info.vpn), hitWay) } 269 270 l3AccessPerf.zip(hitVec).map{ case (l, h) => l := h && RegNext(stage1.fire) } 271 XSDebug(stage1.fire, p"[l3] ridx:0x${Hexadecimal(ridx)}\n") 272 for (i <- 0 until l2tlbParams.l3nWays) { 273 XSDebug(RegNext(stage1.fire), p"[l3] ramDatas(${i.U}) ${ramDatas(i)} l3v:${vidx(i)} hit:${ramDatas(i).entries.hit(stage2(0).bits.req_info.vpn, io.csr.satp.asid)}\n") 274 } 275 XSDebug(stage2(0).valid, p"[l3] l3Hit:${hit} l3HitData:${hitWayData} hitVec:${Binary(hitVec.asUInt)} hitWay:${hitWay} vidx:${Binary(vidx.asUInt)}\n") 276 277 ridx.suggestName(s"l3_ridx") 278 vidx.suggestName(s"l3_vidx") 279 ramDatas.suggestName(s"l3_ramDatas") 280 hitVec.suggestName(s"l3_hitVec") 281 hitWay.suggestName(s"l3_hitWay") 282 283 (hit, hitWayData, hitWayData.prefetch, eccError) 284 } 285 val l3HitPPN = l3HitData.ppns(genPtwL3SectorIdx(stage2(0).bits.req_info.vpn)) 286 val l3HitPerm = l3HitData.perms.getOrElse(0.U.asTypeOf(Vec(PtwL3SectorSize, new PtePermBundle)))(genPtwL3SectorIdx(stage2(0).bits.req_info.vpn)) 287 288 // super page 289 val spreplace = ReplacementPolicy.fromString(l2tlbParams.spReplacer, l2tlbParams.spSize) 290 val (spHit, spHitData, spPre) = { 291 val hitVecT = sp.zipWithIndex.map { case (e, i) => e.hit(stage1.bits.req_info.vpn, io.csr.satp.asid) && spv(i) } 292 val hitVec = hitVecT.map(RegEnable(_, stage1.fire)) 293 val hitData = ParallelPriorityMux(hitVec zip sp) 294 val hit = ParallelOR(hitVec) && cache_read_valid 295 296 when (hit) { spreplace.access(OHToUInt(hitVec)) } 297 298 spAccessPerf.zip(hitVec).map{ case (s, h) => s := h && RegNext(stage1.fire) } 299 for (i <- 0 until l2tlbParams.spSize) { 300 XSDebug(stage1.fire, p"[sp] sp(${i.U}) ${sp(i)} hit:${sp(i).hit(stage1.bits.req_info.vpn, io.csr.satp.asid)} spv:${spv(i)}\n") 301 } 302 XSDebug(stage2(0).valid, p"[sp] spHit:${hit} spHitData:${hitData} hitVec:${Binary(VecInit(hitVec).asUInt)}\n") 303 304 VecInit(hitVecT).suggestName(s"sp_hitVecT") 305 VecInit(hitVec).suggestName(s"sp_hitVec") 306 307 (hit, hitData, hitData.prefetch) 308 } 309 val spHitPerm = spHitData.perm.getOrElse(0.U.asTypeOf(new PtePermBundle)) 310 val spHitLevel = spHitData.level.getOrElse(0.U) 311 312 val s2_res = Wire(new PageCacheRespBundle) 313 s2_res.l1.apply(l1Hit, l1Pre, l1HitPPN) 314 s2_res.l2.apply(l2Hit, l2Pre, l2HitPPN, ecc = l2eccError) 315 s2_res.l3.apply(l3Hit, l3Pre, l3HitPPN, l3HitPerm, l3eccError) 316 s2_res.sp.apply(spHit, spPre, spHitData.ppn, spHitPerm, false.B, spHitLevel) 317 val s2_res_reg = DataHoldBypass(s2_res, RegNext(stage1.fire())) 318 319 // stage3, add stage 3 for ecc check... 320 val s3_res = Reg(new PageCacheRespBundle) 321 when (stage2(1).fire()) { 322 s3_res := s2_res_reg 323 } 324 325 io.resp.bits.req_info := stage3.bits.req_info 326 io.resp.bits.hit := s3_res.l3.hit || s3_res.sp.hit 327 io.resp.bits.prefetch := s3_res.l3.pre && s3_res.l3.hit || s3_res.sp.pre && s3_res.sp.hit 328 io.resp.bits.toFsm.l1Hit := s3_res.l1.hit 329 io.resp.bits.toFsm.l2Hit := s3_res.l2.hit 330 io.resp.bits.toFsm.ppn := Mux(s3_res.l2.hit, s3_res.l2.ppn, s3_res.l1.ppn) 331 io.resp.bits.toTlb.tag := stage3.bits.req_info.vpn 332 io.resp.bits.toTlb.asid := io.csr.satp.asid // DontCare 333 io.resp.bits.toTlb.ppn := Mux(s3_res.l3.hit, s3_res.l3.ppn, s3_res.sp.ppn) 334 io.resp.bits.toTlb.perm.map(_ := Mux(s3_res.l3.hit, s3_res.l3.perm, s3_res.sp.perm)) 335 io.resp.bits.toTlb.level.map(_ := Mux(s3_res.l3.hit, 2.U, s3_res.sp.level)) 336 io.resp.bits.toTlb.prefetch := from_pre(stage3.bits.req_info.source) 337 io.resp.valid := stage3.valid 338 assert(!(l3Hit && spHit), "normal page and super page both hit") 339 340 // refill Perf 341 val l1RefillPerf = Wire(Vec(l2tlbParams.l1Size, Bool())) 342 val l2RefillPerf = Wire(Vec(l2tlbParams.l2nWays, Bool())) 343 val l3RefillPerf = Wire(Vec(l2tlbParams.l3nWays, Bool())) 344 val spRefillPerf = Wire(Vec(l2tlbParams.spSize, Bool())) 345 l1RefillPerf.map(_ := false.B) 346 l2RefillPerf.map(_ := false.B) 347 l3RefillPerf.map(_ := false.B) 348 spRefillPerf.map(_ := false.B) 349 350 // refill 351 l2.io.w.req <> DontCare 352 l3.io.w.req <> DontCare 353 l2.io.w.req.valid := false.B 354 l3.io.w.req.valid := false.B 355 356 def get_part(data: UInt, index: UInt): UInt = { 357 val inner_data = data.asTypeOf(Vec(data.getWidth / XLEN, UInt(XLEN.W))) 358 inner_data(index) 359 } 360 361 val memRdata = refill.ptes 362 val memSelData = get_part(memRdata, refill.addr_low) 363 val memPtes = (0 until (l2tlbParams.blockBytes/(XLEN/8))).map(i => memRdata((i+1)*XLEN-1, i*XLEN).asTypeOf(new PteBundle)) 364 val memPte = memSelData.asTypeOf(new PteBundle) 365 366 memPte.suggestName("memPte") 367 368 // TODO: handle sfenceLatch outsize 369 when (io.refill.valid && !memPte.isPf(refill.level) && !flush ) { 370 when (refill.level === 0.U && !memPte.isLeaf()) { 371 // val refillIdx = LFSR64()(log2Up(l2tlbParams.l1Size)-1,0) // TODO: may be LRU 372 val refillIdx = replaceWrapper(l1v, ptwl1replace.way) 373 refillIdx.suggestName(s"PtwL1RefillIdx") 374 val rfOH = UIntToOH(refillIdx) 375 l1(refillIdx).refill( 376 refill.req_info.vpn, 377 io.csr.satp.asid, 378 memSelData, 379 0.U, 380 refill_prefetch 381 ) 382 ptwl1replace.access(refillIdx) 383 l1v := l1v | rfOH 384 l1g := (l1g & ~rfOH) | Mux(memPte.perm.g, rfOH, 0.U) 385 386 for (i <- 0 until l2tlbParams.l1Size) { 387 l1RefillPerf(i) := i.U === refillIdx 388 } 389 390 XSDebug(p"[l1 refill] refillIdx:${refillIdx} refillEntry:${l1(refillIdx).genPtwEntry(refill.req_info.vpn, io.csr.satp.asid, memSelData, 0.U, refill_prefetch)}\n") 391 XSDebug(p"[l1 refill] l1v:${Binary(l1v)}->${Binary(l1v | rfOH)} l1g:${Binary(l1g)}->${Binary((l1g & ~rfOH) | Mux(memPte.perm.g, rfOH, 0.U))}\n") 392 393 refillIdx.suggestName(s"l1_refillIdx") 394 rfOH.suggestName(s"l1_rfOH") 395 } 396 397 when (refill.level === 1.U && !memPte.isLeaf()) { 398 val refillIdx = genPtwL2SetIdx(refill.req_info.vpn) 399 val victimWay = replaceWrapper(RegEnable(VecInit(getl2vSet(refill.req_info.vpn).asBools).asUInt, stage1.fire), ptwl2replace.way(refillIdx)) 400 val victimWayOH = UIntToOH(victimWay) 401 val rfvOH = UIntToOH(Cat(refillIdx, victimWay)) 402 val wdata = Wire(l2EntryType) 403 wdata.gen( 404 vpn = refill.req_info.vpn, 405 asid = io.csr.satp.asid, 406 data = memRdata, 407 levelUInt = 1.U, 408 refill_prefetch 409 ) 410 l2.io.w.apply( 411 valid = true.B, 412 setIdx = refillIdx, 413 data = wdata, 414 waymask = victimWayOH 415 ) 416 ptwl2replace.access(refillIdx, victimWay) 417 l2v := l2v | rfvOH 418 l2g := l2g & ~rfvOH | Mux(Cat(memPtes.map(_.perm.g)).andR, rfvOH, 0.U) 419 420 for (i <- 0 until l2tlbParams.l2nWays) { 421 l2RefillPerf(i) := i.U === victimWay 422 } 423 424 XSDebug(p"[l2 refill] refillIdx:0x${Hexadecimal(refillIdx)} victimWay:${victimWay} victimWayOH:${Binary(victimWayOH)} rfvOH(in UInt):${Cat(refillIdx, victimWay)}\n") 425 XSDebug(p"[l2 refill] refilldata:0x${wdata}\n") 426 XSDebug(p"[l2 refill] l2v:${Binary(l2v)} -> ${Binary(l2v | rfvOH)}\n") 427 XSDebug(p"[l2 refill] l2g:${Binary(l2g)} -> ${Binary(l2g & ~rfvOH | Mux(Cat(memPtes.map(_.perm.g)).andR, rfvOH, 0.U))}\n") 428 429 refillIdx.suggestName(s"l2_refillIdx") 430 victimWay.suggestName(s"l2_victimWay") 431 victimWayOH.suggestName(s"l2_victimWayOH") 432 rfvOH.suggestName(s"l2_rfvOH") 433 } 434 435 when (refill.level === 2.U && memPte.isLeaf()) { 436 val refillIdx = genPtwL3SetIdx(refill.req_info.vpn) 437 val victimWay = replaceWrapper(RegEnable(VecInit(getl3vSet(refill.req_info.vpn).asBools).asUInt, stage1.fire), ptwl3replace.way(refillIdx)) 438 val victimWayOH = UIntToOH(victimWay) 439 val rfvOH = UIntToOH(Cat(refillIdx, victimWay)) 440 val wdata = Wire(l3EntryType) 441 wdata.gen( 442 vpn = refill.req_info.vpn, 443 asid = io.csr.satp.asid, 444 data = memRdata, 445 levelUInt = 2.U, 446 refill_prefetch 447 ) 448 l3.io.w.apply( 449 valid = true.B, 450 setIdx = refillIdx, 451 data = wdata, 452 waymask = victimWayOH 453 ) 454 ptwl3replace.access(refillIdx, victimWay) 455 l3v := l3v | rfvOH 456 l3g := l3g & ~rfvOH | Mux(Cat(memPtes.map(_.perm.g)).andR, rfvOH, 0.U) 457 458 for (i <- 0 until l2tlbParams.l3nWays) { 459 l3RefillPerf(i) := i.U === victimWay 460 } 461 462 XSDebug(p"[l3 refill] refillIdx:0x${Hexadecimal(refillIdx)} victimWay:${victimWay} victimWayOH:${Binary(victimWayOH)} rfvOH(in UInt):${Cat(refillIdx, victimWay)}\n") 463 XSDebug(p"[l3 refill] refilldata:0x${wdata}\n") 464 XSDebug(p"[l3 refill] l3v:${Binary(l3v)} -> ${Binary(l3v | rfvOH)}\n") 465 XSDebug(p"[l3 refill] l3g:${Binary(l3g)} -> ${Binary(l3g & ~rfvOH | Mux(Cat(memPtes.map(_.perm.g)).andR, rfvOH, 0.U))}\n") 466 467 refillIdx.suggestName(s"l3_refillIdx") 468 victimWay.suggestName(s"l3_victimWay") 469 victimWayOH.suggestName(s"l3_victimWayOH") 470 rfvOH.suggestName(s"l3_rfvOH") 471 } 472 when ((refill.level === 0.U || refill.level === 1.U) && memPte.isLeaf()) { 473 val refillIdx = spreplace.way// LFSR64()(log2Up(l2tlbParams.spSize)-1,0) // TODO: may be LRU 474 val rfOH = UIntToOH(refillIdx) 475 sp(refillIdx).refill( 476 refill.req_info.vpn, 477 io.csr.satp.asid, 478 memSelData, 479 refill.level, 480 refill_prefetch 481 ) 482 spreplace.access(refillIdx) 483 spv := spv | rfOH 484 spg := spg & ~rfOH | Mux(memPte.perm.g, rfOH, 0.U) 485 486 for (i <- 0 until l2tlbParams.spSize) { 487 spRefillPerf(i) := i.U === refillIdx 488 } 489 490 XSDebug(p"[sp refill] refillIdx:${refillIdx} refillEntry:${sp(refillIdx).genPtwEntry(refill.req_info.vpn, io.csr.satp.asid, memSelData, refill.level, refill_prefetch)}\n") 491 XSDebug(p"[sp refill] spv:${Binary(spv)}->${Binary(spv | rfOH)} spg:${Binary(spg)}->${Binary(spg & ~rfOH | Mux(memPte.perm.g, rfOH, 0.U))}\n") 492 493 refillIdx.suggestName(s"sp_refillIdx") 494 rfOH.suggestName(s"sp_rfOH") 495 } 496 } 497 498 val l2eccFlush = s3_res.l2.ecc && stage3.fire() // RegNext(l2eccError, init = false.B) 499 val l3eccFlush = s3_res.l3.ecc && stage3.fire() // RegNext(l3eccError, init = false.B) 500 val eccVpn = stage3.bits.req_info.vpn 501 502 assert(!l2eccFlush) 503 assert(!l3eccFlush) 504 when (l2eccFlush) { 505 val flushSetIdxOH = UIntToOH(genPtwL2SetIdx(eccVpn)) 506 val flushMask = VecInit(flushSetIdxOH.asBools.map { a => Fill(l2tlbParams.l2nWays, a.asUInt) }).asUInt 507 l2v := l2v & ~flushMask 508 l2g := l2g & ~flushMask 509 } 510 511 when (l3eccFlush) { 512 val flushSetIdxOH = UIntToOH(genPtwL3SetIdx(eccVpn)) 513 val flushMask = VecInit(flushSetIdxOH.asBools.map { a => Fill(l2tlbParams.l3nWays, a.asUInt) }).asUInt 514 l3v := l3v & ~flushMask 515 l3g := l3g & ~flushMask 516 } 517 518 // sfence 519 when (sfence.valid) { 520 val l1asidhit = VecInit(l1asids.map(_ === sfence.bits.asid)).asUInt 521 val spasidhit = VecInit(spasids.map(_ === sfence.bits.asid)).asUInt 522 val sfence_vpn = sfence.bits.addr(sfence.bits.addr.getWidth-1, offLen) 523 524 when (sfence.bits.rs1/*va*/) { 525 when (sfence.bits.rs2) { 526 // all va && all asid 527 l1v := 0.U 528 l2v := 0.U 529 l3v := 0.U 530 spv := 0.U 531 } .otherwise { 532 // all va && specific asid except global 533 534 l1v := l1v & (~l1asidhit | l1g) 535 l2v := l2v & l2g 536 l3v := l3v & l3g 537 spv := spv & (~spasidhit | spg) 538 } 539 } .otherwise { 540 // val flushMask = UIntToOH(genTlbL2Idx(sfence.bits.addr(sfence.bits.addr.getWidth-1, offLen))) 541 val flushSetIdxOH = UIntToOH(genPtwL3SetIdx(sfence_vpn)) 542 // val flushMask = VecInit(flushSetIdxOH.asBools.map(Fill(l2tlbParams.l3nWays, _.asUInt))).asUInt 543 val flushMask = VecInit(flushSetIdxOH.asBools.map { a => Fill(l2tlbParams.l3nWays, a.asUInt) }).asUInt 544 flushSetIdxOH.suggestName(s"sfence_nrs1_flushSetIdxOH") 545 flushMask.suggestName(s"sfence_nrs1_flushMask") 546 547 when (sfence.bits.rs2) { 548 // specific leaf of addr && all asid 549 l3v := l3v & ~flushMask 550 spv := spv & (~VecInit(sp.map(_.hit(sfence_vpn, sfence.bits.asid, ignoreAsid = true))).asUInt | spg) 551 } .otherwise { 552 // specific leaf of addr && specific asid 553 l3v := l3v & (~flushMask | l3g) 554 spv := spv & (~VecInit(sp.map(_.hit(sfence_vpn, sfence.bits.asid))).asUInt | spg) 555 } 556 } 557 } 558 559 def InsideStageConnect[T <:Data](in: DecoupledIO[T], out: DecoupledIO[T], block: Bool = false.B): Unit = { 560 in.ready := !in.valid || out.ready 561 out.valid := in.valid 562 out.bits := in.bits 563 } 564 565 // Perf Count 566 val resp_l3 = s3_res.l3.hit 567 val resp_sp = s3_res.sp.hit 568 val resp_l1_pre = s3_res.l1.pre 569 val resp_l2_pre = s3_res.l2.pre 570 val resp_l3_pre = s3_res.l3.pre 571 val resp_sp_pre = s3_res.sp.pre 572 val base_valid_access_0 = !from_pre(io.resp.bits.req_info.source) && io.resp.fire() 573 XSPerfAccumulate("access", base_valid_access_0) 574 XSPerfAccumulate("l1_hit", base_valid_access_0 && io.resp.bits.toFsm.l1Hit && !io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 575 XSPerfAccumulate("l2_hit", base_valid_access_0 && io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 576 XSPerfAccumulate("l3_hit", base_valid_access_0 && resp_l3) 577 XSPerfAccumulate("sp_hit", base_valid_access_0 && resp_sp) 578 XSPerfAccumulate("pte_hit",base_valid_access_0 && io.resp.bits.hit) 579 580 XSPerfAccumulate("l1_hit_pre", base_valid_access_0 && resp_l1_pre && io.resp.bits.toFsm.l1Hit && !io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 581 XSPerfAccumulate("l2_hit_pre", base_valid_access_0 && resp_l2_pre && io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 582 XSPerfAccumulate("l3_hit_pre", base_valid_access_0 && resp_l3_pre && resp_l3) 583 XSPerfAccumulate("sp_hit_pre", base_valid_access_0 && resp_sp_pre && resp_sp) 584 XSPerfAccumulate("pte_hit_pre",base_valid_access_0 && (resp_l3_pre && resp_l3 || resp_sp_pre && resp_sp) && io.resp.bits.hit) 585 586 val base_valid_access_1 = from_pre(io.resp.bits.req_info.source) && io.resp.fire() 587 XSPerfAccumulate("pre_access", base_valid_access_1) 588 XSPerfAccumulate("pre_l1_hit", base_valid_access_1 && io.resp.bits.toFsm.l1Hit && !io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 589 XSPerfAccumulate("pre_l2_hit", base_valid_access_1 && io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 590 XSPerfAccumulate("pre_l3_hit", base_valid_access_1 && resp_l3) 591 XSPerfAccumulate("pre_sp_hit", base_valid_access_1 && resp_sp) 592 XSPerfAccumulate("pre_pte_hit",base_valid_access_1 && io.resp.bits.hit) 593 594 XSPerfAccumulate("pre_l1_hit_pre", base_valid_access_1 && resp_l1_pre && io.resp.bits.toFsm.l1Hit && !io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 595 XSPerfAccumulate("pre_l2_hit_pre", base_valid_access_1 && resp_l2_pre && io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 596 XSPerfAccumulate("pre_l3_hit_pre", base_valid_access_1 && resp_l3_pre && resp_l3) 597 XSPerfAccumulate("pre_sp_hit_pre", base_valid_access_1 && resp_sp_pre && resp_sp) 598 XSPerfAccumulate("pre_pte_hit_pre",base_valid_access_1 && (resp_l3_pre && resp_l3 || resp_sp_pre && resp_sp) && io.resp.bits.hit) 599 600 val base_valid_access_2 = stage3.bits.isFirst && !from_pre(io.resp.bits.req_info.source) && io.resp.fire() 601 XSPerfAccumulate("access_first", base_valid_access_2) 602 XSPerfAccumulate("l1_hit_first", base_valid_access_2 && io.resp.bits.toFsm.l1Hit && !io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 603 XSPerfAccumulate("l2_hit_first", base_valid_access_2 && io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 604 XSPerfAccumulate("l3_hit_first", base_valid_access_2 && resp_l3) 605 XSPerfAccumulate("sp_hit_first", base_valid_access_2 && resp_sp) 606 XSPerfAccumulate("pte_hit_first",base_valid_access_2 && io.resp.bits.hit) 607 608 XSPerfAccumulate("l1_hit_pre_first", base_valid_access_2 && resp_l1_pre && io.resp.bits.toFsm.l1Hit && !io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 609 XSPerfAccumulate("l2_hit_pre_first", base_valid_access_2 && resp_l2_pre && io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 610 XSPerfAccumulate("l3_hit_pre_first", base_valid_access_2 && resp_l3_pre && resp_l3) 611 XSPerfAccumulate("sp_hit_pre_first", base_valid_access_2 && resp_sp_pre && resp_sp) 612 XSPerfAccumulate("pte_hit_pre_first",base_valid_access_2 && (resp_l3_pre && resp_l3 || resp_sp_pre && resp_sp) && io.resp.bits.hit) 613 614 val base_valid_access_3 = stage3.bits.isFirst && from_pre(io.resp.bits.req_info.source) && io.resp.fire() 615 XSPerfAccumulate("pre_access_first", base_valid_access_3) 616 XSPerfAccumulate("pre_l1_hit_first", base_valid_access_3 && io.resp.bits.toFsm.l1Hit && !io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 617 XSPerfAccumulate("pre_l2_hit_first", base_valid_access_3 && io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 618 XSPerfAccumulate("pre_l3_hit_first", base_valid_access_3 && resp_l3) 619 XSPerfAccumulate("pre_sp_hit_first", base_valid_access_3 && resp_sp) 620 XSPerfAccumulate("pre_pte_hit_first", base_valid_access_3 && io.resp.bits.hit) 621 622 XSPerfAccumulate("pre_l1_hit_pre_first", base_valid_access_3 && resp_l1_pre && io.resp.bits.toFsm.l1Hit && !io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 623 XSPerfAccumulate("pre_l2_hit_pre_first", base_valid_access_3 && resp_l2_pre && io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 624 XSPerfAccumulate("pre_l3_hit_pre_first", base_valid_access_3 && resp_l3_pre && resp_l3) 625 XSPerfAccumulate("pre_sp_hit_pre_first", base_valid_access_3 && resp_sp_pre && resp_sp) 626 XSPerfAccumulate("pre_pte_hit_pre_first",base_valid_access_3 && (resp_l3_pre && resp_l3 || resp_sp_pre && resp_sp) && io.resp.bits.hit) 627 628 XSPerfAccumulate("rwHarzad", io.req.valid && !io.req.ready) 629 XSPerfAccumulate("out_blocked", io.resp.valid && !io.resp.ready) 630 l1AccessPerf.zipWithIndex.map{ case (l, i) => XSPerfAccumulate(s"L1AccessIndex${i}", l) } 631 l2AccessPerf.zipWithIndex.map{ case (l, i) => XSPerfAccumulate(s"L2AccessIndex${i}", l) } 632 l3AccessPerf.zipWithIndex.map{ case (l, i) => XSPerfAccumulate(s"L3AccessIndex${i}", l) } 633 spAccessPerf.zipWithIndex.map{ case (l, i) => XSPerfAccumulate(s"SPAccessIndex${i}", l) } 634 l1RefillPerf.zipWithIndex.map{ case (l, i) => XSPerfAccumulate(s"L1RefillIndex${i}", l) } 635 l2RefillPerf.zipWithIndex.map{ case (l, i) => XSPerfAccumulate(s"L2RefillIndex${i}", l) } 636 l3RefillPerf.zipWithIndex.map{ case (l, i) => XSPerfAccumulate(s"L3RefillIndex${i}", l) } 637 spRefillPerf.zipWithIndex.map{ case (l, i) => XSPerfAccumulate(s"SPRefillIndex${i}", l) } 638 639 XSPerfAccumulate("l1Refill", Cat(l1RefillPerf).orR) 640 XSPerfAccumulate("l2Refill", Cat(l2RefillPerf).orR) 641 XSPerfAccumulate("l3Refill", Cat(l3RefillPerf).orR) 642 XSPerfAccumulate("spRefill", Cat(spRefillPerf).orR) 643 XSPerfAccumulate("l1Refill_pre", Cat(l1RefillPerf).orR && refill_prefetch) 644 XSPerfAccumulate("l2Refill_pre", Cat(l2RefillPerf).orR && refill_prefetch) 645 XSPerfAccumulate("l3Refill_pre", Cat(l3RefillPerf).orR && refill_prefetch) 646 XSPerfAccumulate("spRefill_pre", Cat(spRefillPerf).orR && refill_prefetch) 647 648 // debug 649 XSDebug(sfence.valid, p"[sfence] original v and g vector:\n") 650 XSDebug(sfence.valid, p"[sfence] l1v:${Binary(l1v)}\n") 651 XSDebug(sfence.valid, p"[sfence] l2v:${Binary(l2v)}\n") 652 XSDebug(sfence.valid, p"[sfence] l3v:${Binary(l3v)}\n") 653 XSDebug(sfence.valid, p"[sfence] l3g:${Binary(l3g)}\n") 654 XSDebug(sfence.valid, p"[sfence] spv:${Binary(spv)}\n") 655 XSDebug(RegNext(sfence.valid), p"[sfence] new v and g vector:\n") 656 XSDebug(RegNext(sfence.valid), p"[sfence] l1v:${Binary(l1v)}\n") 657 XSDebug(RegNext(sfence.valid), p"[sfence] l2v:${Binary(l2v)}\n") 658 XSDebug(RegNext(sfence.valid), p"[sfence] l3v:${Binary(l3v)}\n") 659 XSDebug(RegNext(sfence.valid), p"[sfence] l3g:${Binary(l3g)}\n") 660 XSDebug(RegNext(sfence.valid), p"[sfence] spv:${Binary(spv)}\n") 661 662 val perfinfo = IO(new Bundle(){ 663 val perfEvents = Output(new PerfEventsBundle(8)) 664 }) 665 val perfEvents = Seq( 666 ("access ", base_valid_access_0 ), 667 ("l1_hit ", l1Hit ), 668 ("l2_hit ", l2Hit ), 669 ("l3_hit ", l3Hit ), 670 ("sp_hit ", spHit ), 671 ("pte_hit ", l3Hit || spHit ), 672 ("rwHarzad ", io.req.valid && !io.req.ready ), 673 ("out_blocked ", io.resp.valid && !io.resp.ready), 674 ) 675 676 for (((perf_out,(perf_name,perf)),i) <- perfinfo.perfEvents.perf_events.zip(perfEvents).zipWithIndex) { 677 perf_out.incr_step := RegNext(perf) 678 } 679} 680