xref: /XiangShan/src/main/scala/xiangshan/cache/mmu/PageTableCache.scala (revision 44af22172d764d72860feb70b7d6aa9819594d79) 
1/***************************************************************************************
2* Copyright (c) 2024 Beijing Institute of Open Source Chip (BOSC)
3* Copyright (c) 2020-2024 Institute of Computing Technology, Chinese Academy of Sciences
4* Copyright (c) 2020-2021 Peng Cheng Laboratory
5*
6* XiangShan is licensed under Mulan PSL v2.
7* You can use this software according to the terms and conditions of the Mulan PSL v2.
8* You may obtain a copy of Mulan PSL v2 at:
9*          http://license.coscl.org.cn/MulanPSL2
10*
11* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
12* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
13* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
14*
15* See the Mulan PSL v2 for more details.
16***************************************************************************************/
17
18package xiangshan.cache.mmu
19
20import org.chipsalliance.cde.config.Parameters
21import chisel3._
22import chisel3.util._
23import xiangshan._
24import xiangshan.cache.{HasDCacheParameters, MemoryOpConstants}
25import utils._
26import utility._
27import freechips.rocketchip.diplomacy.{LazyModule, LazyModuleImp}
28import freechips.rocketchip.tilelink._
29
30/* ptw cache caches the page table of all the three layers
31 * ptw cache resp at next cycle
32 * the cache should not be blocked
33 * when miss queue if full, just block req outside
34 */
35
36class PageCachePerPespBundle(implicit p: Parameters) extends PtwBundle {
37  val hit = Bool()
38  val pre = Bool()
39  val ppn = UInt(gvpnLen.W)
40  val perm = new PtePermBundle()
41  val ecc = Bool()
42  val level = UInt(2.W)
43  val v = Bool()
44
45  def apply(hit: Bool, pre: Bool, ppn: UInt, perm: PtePermBundle = 0.U.asTypeOf(new PtePermBundle()),
46            ecc: Bool = false.B, level: UInt = 0.U, valid: Bool = true.B): Unit = {
47    this.hit := hit && !ecc
48    this.pre := pre
49    this.ppn := ppn
50    this.perm := perm
51    this.ecc := ecc && hit
52    this.level := level
53    this.v := valid
54  }
55}
56
57class PageCacheMergePespBundle(implicit p: Parameters) extends PtwBundle {
58  assert(tlbcontiguous == 8, "Only support tlbcontiguous = 8!")
59  val hit = Bool()
60  val pre = Bool()
61  val ppn = Vec(tlbcontiguous, UInt(gvpnLen.W))
62  val perm = Vec(tlbcontiguous, new PtePermBundle())
63  val ecc = Bool()
64  val level = UInt(2.W)
65  val v = Vec(tlbcontiguous, Bool())
66  val af = Vec(tlbcontiguous, Bool())
67
68  def apply(hit: Bool, pre: Bool, ppn: Vec[UInt], perm: Vec[PtePermBundle] = Vec(tlbcontiguous, 0.U.asTypeOf(new PtePermBundle())),
69            ecc: Bool = false.B, level: UInt = 0.U, valid: Vec[Bool] = Vec(tlbcontiguous, true.B), accessFault: Vec[Bool] = Vec(tlbcontiguous, true.B)): Unit = {
70    this.hit := hit && !ecc
71    this.pre := pre
72    this.ppn := ppn
73    this.perm := perm
74    this.ecc := ecc && hit
75    this.level := level
76    this.v := valid
77    this.af := accessFault
78  }
79}
80
81class PageCacheRespBundle(implicit p: Parameters) extends PtwBundle {
82  val l1 = new PageCachePerPespBundle
83  val l2 = new PageCachePerPespBundle
84  val l3 = new PageCacheMergePespBundle
85  val sp = new PageCachePerPespBundle
86}
87
88class PtwCacheReq(implicit p: Parameters) extends PtwBundle {
89  val req_info = new L2TlbInnerBundle()
90  val isFirst = Bool()
91  val bypassed = Vec(3, Bool())
92  val isHptwReq = Bool()
93  val hptwId = UInt(log2Up(l2tlbParams.llptwsize).W)
94}
95
96class PtwCacheIO()(implicit p: Parameters) extends MMUIOBaseBundle with HasPtwConst {
97  val req = Flipped(DecoupledIO(new PtwCacheReq()))
98  val resp = DecoupledIO(new Bundle {
99    val req_info = new L2TlbInnerBundle()
100    val isFirst = Bool()
101    val hit = Bool()
102    val prefetch = Bool() // is the entry fetched by prefetch
103    val bypassed = Bool()
104    val toFsm = new Bundle {
105      val l1Hit = Bool()
106      val l2Hit = Bool()
107      val ppn = UInt(gvpnLen.W)
108      val stage1Hit = Bool() // find stage 1 pte in cache, but need to search stage 2 pte in cache at PTW
109    }
110    val stage1 = new PtwMergeResp()
111    val isHptwReq = Bool()
112    val toHptw = new Bundle {
113      val l1Hit = Bool()
114      val l2Hit = Bool()
115      val ppn = UInt(ppnLen.W)
116      val id = UInt(log2Up(l2tlbParams.llptwsize).W)
117      val resp = new HptwResp() // used if hit
118      val bypassed = Bool()
119    }
120  })
121  val refill = Flipped(ValidIO(new Bundle {
122    val ptes = UInt(blockBits.W)
123    val levelOH = new Bundle {
124      // NOTE: levelOH has (Level+1) bits, each stands for page cache entries
125      val sp = Bool()
126      val l3 = Bool()
127      val l2 = Bool()
128      val l1 = Bool()
129      def apply(levelUInt: UInt, valid: Bool) = {
130        sp := GatedValidRegNext((levelUInt === 0.U || levelUInt === 1.U) && valid, false.B)
131        l3 := GatedValidRegNext((levelUInt === 2.U) & valid, false.B)
132        l2 := GatedValidRegNext((levelUInt === 1.U) & valid, false.B)
133        l1 := GatedValidRegNext((levelUInt === 0.U) & valid, false.B)
134      }
135    }
136    // duplicate level and sel_pte for each page caches, for better fanout
137    val req_info_dup = Vec(3, new L2TlbInnerBundle())
138    val level_dup = Vec(3, UInt(log2Up(Level).W))
139    val sel_pte_dup = Vec(3, UInt(XLEN.W))
140  }))
141  val sfence_dup = Vec(4, Input(new SfenceBundle()))
142  val csr_dup = Vec(3, Input(new TlbCsrBundle()))
143}
144
145class PtwCache()(implicit p: Parameters) extends XSModule with HasPtwConst with HasPerfEvents {
146  val io = IO(new PtwCacheIO)
147  val ecc = Code.fromString(l2tlbParams.ecc)
148  val l2EntryType = new PTWEntriesWithEcc(ecc, num = PtwL2SectorSize, tagLen = PtwL2TagLen, level = 1, hasPerm = false, hasReservedBitforMbist = true)
149  val l3EntryType = new PTWEntriesWithEcc(ecc, num = PtwL3SectorSize, tagLen = PtwL3TagLen, level = 2, hasPerm = true)
150
151  // TODO: four caches make the codes dirty, think about how to deal with it
152
153  val sfence_dup = io.sfence_dup
154  val refill = io.refill.bits
155  val refill_prefetch_dup = io.refill.bits.req_info_dup.map(a => from_pre(a.source))
156  val refill_h = io.refill.bits.req_info_dup.map(a => Mux(a.s2xlate === allStage, onlyStage1, a.s2xlate))
157  val flush_dup = sfence_dup.zip(io.csr_dup).map(f => f._1.valid || f._2.satp.changed || f._2.vsatp.changed || f._2.hgatp.changed)
158  val flush = flush_dup(0)
159
160  // when refill, refuce to accept new req
161  val rwHarzad = if (sramSinglePort) io.refill.valid else false.B
162
163  // handle hand signal and req_info
164  // TODO: replace with FlushableQueue
165  val stageReq = Wire(Decoupled(new PtwCacheReq()))         // enq stage & read page cache valid
166  val stageDelay = Wire(Vec(2, Decoupled(new PtwCacheReq()))) // page cache resp
167  val stageCheck = Wire(Vec(2, Decoupled(new PtwCacheReq()))) // check hit & check ecc
168  val stageResp = Wire(Decoupled(new PtwCacheReq()))         // deq stage
169
170  val stageDelay_valid_1cycle = OneCycleValid(stageReq.fire, flush)      // catch ram data
171  val stageCheck_valid_1cycle = OneCycleValid(stageDelay(1).fire, flush) // replace & perf counter
172  val stageResp_valid_1cycle_dup = Wire(Vec(2, Bool()))
173  stageResp_valid_1cycle_dup.map(_ := OneCycleValid(stageCheck(1).fire, flush))  // ecc flush
174
175  stageReq <> io.req
176  PipelineConnect(stageReq, stageDelay(0), stageDelay(1).ready, flush, rwHarzad)
177  InsideStageConnect(stageDelay(0), stageDelay(1), stageDelay_valid_1cycle)
178  PipelineConnect(stageDelay(1), stageCheck(0), stageCheck(1).ready, flush)
179  InsideStageConnect(stageCheck(0), stageCheck(1), stageCheck_valid_1cycle)
180  PipelineConnect(stageCheck(1), stageResp, io.resp.ready, flush)
181  stageResp.ready := !stageResp.valid || io.resp.ready
182
183  // l1: level 0 non-leaf pte
184  val l1 = Reg(Vec(l2tlbParams.l1Size, new PtwEntry(tagLen = PtwL1TagLen)))
185  val l1v = RegInit(0.U(l2tlbParams.l1Size.W))
186  val l1g = Reg(UInt(l2tlbParams.l1Size.W))
187  val l1asids = l1.map(_.asid)
188  val l1vmids = l1.map(_.vmid)
189  val l1h = Reg(Vec(l2tlbParams.l1Size, UInt(2.W)))
190
191  // l2: level 1 non-leaf pte
192  val l2 = Module(new SRAMTemplate(
193    l2EntryType,
194    set = l2tlbParams.l2nSets,
195    way = l2tlbParams.l2nWays,
196    singlePort = sramSinglePort
197  ))
198  val l2v = RegInit(0.U((l2tlbParams.l2nSets * l2tlbParams.l2nWays).W))
199  val l2g = Reg(UInt((l2tlbParams.l2nSets * l2tlbParams.l2nWays).W))
200  val l2h = Reg(Vec(l2tlbParams.l2nSets, Vec(l2tlbParams.l2nWays, UInt(2.W))))
201  def getl2vSet(vpn: UInt) = {
202    require(log2Up(l2tlbParams.l2nWays) == log2Down(l2tlbParams.l2nWays))
203    val set = genPtwL2SetIdx(vpn)
204    require(set.getWidth == log2Up(l2tlbParams.l2nSets))
205    val l2vVec = l2v.asTypeOf(Vec(l2tlbParams.l2nSets, UInt(l2tlbParams.l2nWays.W)))
206    l2vVec(set)
207  }
208  def getl2hSet(vpn: UInt) = {
209    require(log2Up(l2tlbParams.l2nWays) == log2Down(l2tlbParams.l2nWays))
210    val set = genPtwL2SetIdx(vpn)
211    require(set.getWidth == log2Up(l2tlbParams.l2nSets))
212    l2h(set)
213  }
214
215
216
217  // l3: level 2 leaf pte of 4KB pages
218  val l3 = Module(new SRAMTemplate(
219    l3EntryType,
220    set = l2tlbParams.l3nSets,
221    way = l2tlbParams.l3nWays,
222    singlePort = sramSinglePort
223  ))
224  val l3v = RegInit(0.U((l2tlbParams.l3nSets * l2tlbParams.l3nWays).W))
225  val l3g = Reg(UInt((l2tlbParams.l3nSets * l2tlbParams.l3nWays).W))
226  val l3h = Reg(Vec(l2tlbParams.l3nSets, Vec(l2tlbParams.l3nWays, UInt(2.W))))
227  def getl3vSet(vpn: UInt) = {
228    require(log2Up(l2tlbParams.l3nWays) == log2Down(l2tlbParams.l3nWays))
229    val set = genPtwL3SetIdx(vpn)
230    require(set.getWidth == log2Up(l2tlbParams.l3nSets))
231    val l3vVec = l3v.asTypeOf(Vec(l2tlbParams.l3nSets, UInt(l2tlbParams.l3nWays.W)))
232    l3vVec(set)
233  }
234  def getl3hSet(vpn: UInt) = {
235    require(log2Up(l2tlbParams.l3nWays) == log2Down(l2tlbParams.l3nWays))
236    val set = genPtwL3SetIdx(vpn)
237    require(set.getWidth == log2Up(l2tlbParams.l3nSets))
238    l3h(set)
239  }
240
241  // sp: level 0/1 leaf pte of 1GB/2MB super pages
242  val sp = Reg(Vec(l2tlbParams.spSize, new PtwEntry(tagLen = SPTagLen, hasPerm = true, hasLevel = true)))
243  val spv = RegInit(0.U(l2tlbParams.spSize.W))
244  val spg = Reg(UInt(l2tlbParams.spSize.W))
245  val spasids = sp.map(_.asid)
246  val spvmids = sp.map(_.vmid)
247  val sph = Reg(Vec(l2tlbParams.spSize, UInt(2.W)))
248
249  // Access Perf
250  val l1AccessPerf = Wire(Vec(l2tlbParams.l1Size, Bool()))
251  val l2AccessPerf = Wire(Vec(l2tlbParams.l2nWays, Bool()))
252  val l3AccessPerf = Wire(Vec(l2tlbParams.l3nWays, Bool()))
253  val spAccessPerf = Wire(Vec(l2tlbParams.spSize, Bool()))
254  l1AccessPerf.map(_ := false.B)
255  l2AccessPerf.map(_ := false.B)
256  l3AccessPerf.map(_ := false.B)
257  spAccessPerf.map(_ := false.B)
258
259
260
261  def vpn_match(vpn1: UInt, vpn2: UInt, level: Int) = {
262    (vpn1(vpnLen-1, vpnnLen*(2-level)+3) === vpn2(vpnLen-1, vpnnLen*(2-level)+3))
263  }
264  // NOTE: not actually bypassed, just check if hit, re-access the page cache
265  def refill_bypass(vpn: UInt, level: Int, h_search: UInt) = {
266    val change_h = MuxLookup(h_search, noS2xlate)(Seq(
267      allStage -> onlyStage1,
268      onlyStage1 -> onlyStage1,
269      onlyStage2 -> onlyStage2
270    ))
271    val change_refill_h = MuxLookup(io.refill.bits.req_info_dup(0).s2xlate, noS2xlate)(Seq(
272      allStage -> onlyStage1,
273      onlyStage1 -> onlyStage1,
274      onlyStage2 -> onlyStage2
275    ))
276    val refill_vpn = io.refill.bits.req_info_dup(0).vpn
277    io.refill.valid && (level.U === io.refill.bits.level_dup(0)) && vpn_match(refill_vpn, vpn, level) && change_h === change_refill_h
278  }
279
280  val vpn_search = stageReq.bits.req_info.vpn
281  val h_search = MuxLookup(stageReq.bits.req_info.s2xlate, noS2xlate)(Seq(
282    allStage -> onlyStage1,
283    onlyStage1 -> onlyStage1,
284    onlyStage2 -> onlyStage2
285  ))
286  // l1
287  val ptwl1replace = ReplacementPolicy.fromString(l2tlbParams.l1Replacer, l2tlbParams.l1Size)
288  val (l1Hit, l1HitPPN, l1Pre) = {
289    val hitVecT = l1.zipWithIndex.map {
290      case (e, i) => (e.hit(vpn_search, io.csr_dup(0).satp.asid, io.csr_dup(0).vsatp.asid, io.csr_dup(0).hgatp.asid, s2xlate = h_search =/= noS2xlate)
291        && l1v(i) && h_search === l1h(i))
292    }
293    val hitVec = hitVecT.map(RegEnable(_, stageReq.fire))
294
295    // stageDelay, but check for l1
296    val hitPPN = DataHoldBypass(ParallelPriorityMux(hitVec zip l1.map(_.ppn)), stageDelay_valid_1cycle)
297    val hitPre = DataHoldBypass(ParallelPriorityMux(hitVec zip l1.map(_.prefetch)), stageDelay_valid_1cycle)
298    val hit = DataHoldBypass(ParallelOR(hitVec), stageDelay_valid_1cycle)
299
300    when (hit && stageDelay_valid_1cycle) { ptwl1replace.access(OHToUInt(hitVec)) }
301
302    l1AccessPerf.zip(hitVec).map{ case (l, h) => l := h && stageDelay_valid_1cycle}
303    for (i <- 0 until l2tlbParams.l1Size) {
304      XSDebug(stageReq.fire, p"[l1] l1(${i.U}) ${l1(i)} hit:${l1(i).hit(vpn_search, io.csr_dup(0).satp.asid, io.csr_dup(0).vsatp.asid, io.csr_dup(0).hgatp.asid, s2xlate = h_search =/= noS2xlate)}\n")
305    }
306    XSDebug(stageReq.fire, p"[l1] l1v:${Binary(l1v)} hitVecT:${Binary(VecInit(hitVecT).asUInt)}\n")
307    XSDebug(stageDelay(0).valid, p"[l1] l1Hit:${hit} l1HitPPN:0x${Hexadecimal(hitPPN)} hitVec:${VecInit(hitVec).asUInt}\n")
308
309    VecInit(hitVecT).suggestName(s"l1_hitVecT")
310    VecInit(hitVec).suggestName(s"l1_hitVec")
311
312    // synchronize with other entries with RegEnable
313    (RegEnable(hit, stageDelay(1).fire),
314     RegEnable(hitPPN, stageDelay(1).fire),
315     RegEnable(hitPre, stageDelay(1).fire))
316  }
317
318  // l2
319  val ptwl2replace = ReplacementPolicy.fromString(l2tlbParams.l2Replacer,l2tlbParams.l2nWays,l2tlbParams.l2nSets)
320  val (l2Hit, l2HitPPN, l2Pre, l2eccError) = {
321    val ridx = genPtwL2SetIdx(vpn_search)
322    l2.io.r.req.valid := stageReq.fire
323    l2.io.r.req.bits.apply(setIdx = ridx)
324    val vVec_req = getl2vSet(vpn_search)
325    val hVec_req = getl2hSet(vpn_search)
326
327    // delay one cycle after sram read
328    val delay_vpn = stageDelay(0).bits.req_info.vpn
329    val delay_h = MuxLookup(stageDelay(0).bits.req_info.s2xlate, noS2xlate)(Seq(
330      allStage -> onlyStage1,
331      onlyStage1 -> onlyStage1,
332      onlyStage2 -> onlyStage2
333    ))
334    val data_resp = DataHoldBypass(l2.io.r.resp.data, stageDelay_valid_1cycle)
335    val vVec_delay = RegEnable(vVec_req, stageReq.fire)
336    val hVec_delay = RegEnable(hVec_req, stageReq.fire)
337    val hitVec_delay = VecInit(data_resp.zip(vVec_delay.asBools).zip(hVec_delay).map { case ((wayData, v), h) =>
338      wayData.entries.hit(delay_vpn, io.csr_dup(1).satp.asid, io.csr_dup(1).vsatp.asid, io.csr_dup(1).hgatp.asid, s2xlate = delay_h =/= noS2xlate) && v && (delay_h === h)})
339
340    // check hit and ecc
341    val check_vpn = stageCheck(0).bits.req_info.vpn
342    val ramDatas = RegEnable(data_resp, stageDelay(1).fire)
343    val vVec = RegEnable(vVec_delay, stageDelay(1).fire).asBools
344
345    val hitVec = RegEnable(hitVec_delay, stageDelay(1).fire)
346    val hitWayEntry = ParallelPriorityMux(hitVec zip ramDatas)
347    val hitWayData = hitWayEntry.entries
348    val hit = ParallelOR(hitVec)
349    val hitWay = ParallelPriorityMux(hitVec zip (0 until l2tlbParams.l2nWays).map(_.U(log2Up(l2tlbParams.l2nWays).W)))
350    val eccError = WireInit(false.B)
351    if (l2tlbParams.enablePTWECC) {
352      eccError := hitWayEntry.decode()
353    } else {
354      eccError := false.B
355    }
356
357    ridx.suggestName(s"l2_ridx")
358    ramDatas.suggestName(s"l2_ramDatas")
359    hitVec.suggestName(s"l2_hitVec")
360    hitWayData.suggestName(s"l2_hitWayData")
361    hitWay.suggestName(s"l2_hitWay")
362
363    when (hit && stageCheck_valid_1cycle) { ptwl2replace.access(genPtwL2SetIdx(check_vpn), hitWay) }
364
365    l2AccessPerf.zip(hitVec).map{ case (l, h) => l := h && stageCheck_valid_1cycle }
366    XSDebug(stageDelay_valid_1cycle, p"[l2] ridx:0x${Hexadecimal(ridx)}\n")
367    for (i <- 0 until l2tlbParams.l2nWays) {
368      XSDebug(stageCheck_valid_1cycle, p"[l2] ramDatas(${i.U}) ${ramDatas(i)}  l2v:${vVec(i)}  hit:${hit}\n")
369    }
370    XSDebug(stageCheck_valid_1cycle, p"[l2] l2Hit:${hit} l2HitPPN:0x${Hexadecimal(hitWayData.ppns(genPtwL2SectorIdx(check_vpn)))} hitVec:${Binary(hitVec.asUInt)} hitWay:${hitWay} vidx:${vVec}\n")
371
372    (hit, hitWayData.ppns(genPtwL2SectorIdx(check_vpn)), hitWayData.prefetch, eccError)
373  }
374
375  // l3
376  val ptwl3replace = ReplacementPolicy.fromString(l2tlbParams.l3Replacer,l2tlbParams.l3nWays,l2tlbParams.l3nSets)
377  val (l3Hit, l3HitData, l3Pre, l3eccError) = {
378    val ridx = genPtwL3SetIdx(vpn_search)
379    l3.io.r.req.valid := stageReq.fire
380    l3.io.r.req.bits.apply(setIdx = ridx)
381    val vVec_req = getl3vSet(vpn_search)
382    val hVec_req = getl3hSet(vpn_search)
383
384    // delay one cycle after sram read
385    val delay_vpn = stageDelay(0).bits.req_info.vpn
386    val delay_h = MuxLookup(stageDelay(0).bits.req_info.s2xlate, noS2xlate)(Seq(
387      allStage -> onlyStage1,
388      onlyStage1 -> onlyStage1,
389      onlyStage2 -> onlyStage2
390    ))
391    val data_resp = DataHoldBypass(l3.io.r.resp.data, stageDelay_valid_1cycle)
392    val vVec_delay = RegEnable(vVec_req, stageReq.fire)
393    val hVec_delay = RegEnable(hVec_req, stageReq.fire)
394    val hitVec_delay = VecInit(data_resp.zip(vVec_delay.asBools).zip(hVec_delay).map { case ((wayData, v), h) =>
395      wayData.entries.hit(delay_vpn, io.csr_dup(2).satp.asid, io.csr_dup(2).vsatp.asid, io.csr_dup(2).hgatp.asid, s2xlate = delay_h =/= noS2xlate) && v && (delay_h === h)})
396
397    // check hit and ecc
398    val check_vpn = stageCheck(0).bits.req_info.vpn
399    val ramDatas = RegEnable(data_resp, stageDelay(1).fire)
400    val vVec = RegEnable(vVec_delay, stageDelay(1).fire).asBools
401
402    val hitVec = RegEnable(hitVec_delay, stageDelay(1).fire)
403    val hitWayEntry = ParallelPriorityMux(hitVec zip ramDatas)
404    val hitWayData = hitWayEntry.entries
405    val hitWayEcc = hitWayEntry.ecc
406    val hit = ParallelOR(hitVec)
407    val hitWay = ParallelPriorityMux(hitVec zip (0 until l2tlbParams.l3nWays).map(_.U(log2Up(l2tlbParams.l3nWays).W)))
408    val eccError = WireInit(false.B)
409    if (l2tlbParams.enablePTWECC) {
410      eccError := hitWayEntry.decode()
411    } else {
412      eccError := false.B
413    }
414
415    when (hit && stageCheck_valid_1cycle) { ptwl3replace.access(genPtwL3SetIdx(check_vpn), hitWay) }
416
417    l3AccessPerf.zip(hitVec).map{ case (l, h) => l := h && stageCheck_valid_1cycle }
418    XSDebug(stageReq.fire, p"[l3] ridx:0x${Hexadecimal(ridx)}\n")
419    for (i <- 0 until l2tlbParams.l3nWays) {
420      XSDebug(stageCheck_valid_1cycle, p"[l3] ramDatas(${i.U}) ${ramDatas(i)}  l3v:${vVec(i)}  hit:${hitVec(i)}\n")
421    }
422    XSDebug(stageCheck_valid_1cycle, p"[l3] l3Hit:${hit} l3HitData:${hitWayData} hitVec:${Binary(hitVec.asUInt)} hitWay:${hitWay} v:${vVec}\n")
423
424    ridx.suggestName(s"l3_ridx")
425    ramDatas.suggestName(s"l3_ramDatas")
426    hitVec.suggestName(s"l3_hitVec")
427    hitWay.suggestName(s"l3_hitWay")
428
429    (hit, hitWayData, hitWayData.prefetch, eccError)
430  }
431  val l3HitPPN = l3HitData.ppns
432  val l3HitPerm = l3HitData.perms.getOrElse(0.U.asTypeOf(Vec(PtwL3SectorSize, new PtePermBundle)))
433  val l3HitValid = l3HitData.vs
434  val l3HitAf = l3HitData.af
435
436  // super page
437  val spreplace = ReplacementPolicy.fromString(l2tlbParams.spReplacer, l2tlbParams.spSize)
438  val (spHit, spHitData, spPre, spValid) = {
439    val hitVecT = sp.zipWithIndex.map { case (e, i) => e.hit(vpn_search, io.csr_dup(0).satp.asid, io.csr_dup(0).vsatp.asid, io.csr_dup(0).hgatp.asid, s2xlate = h_search =/= noS2xlate) && spv(i) && (sph(i) === h_search) }
440    val hitVec = hitVecT.map(RegEnable(_, stageReq.fire))
441    val hitData = ParallelPriorityMux(hitVec zip sp)
442    val hit = ParallelOR(hitVec)
443
444    when (hit && stageDelay_valid_1cycle) { spreplace.access(OHToUInt(hitVec)) }
445
446    spAccessPerf.zip(hitVec).map{ case (s, h) => s := h && stageDelay_valid_1cycle }
447    for (i <- 0 until l2tlbParams.spSize) {
448      XSDebug(stageReq.fire, p"[sp] sp(${i.U}) ${sp(i)} hit:${sp(i).hit(vpn_search, io.csr_dup(0).satp.asid, io.csr_dup(0).vsatp.asid, io.csr_dup(0).hgatp.asid, s2xlate = h_search =/= noS2xlate)} spv:${spv(i)}\n")
449    }
450    XSDebug(stageDelay_valid_1cycle, p"[sp] spHit:${hit} spHitData:${hitData} hitVec:${Binary(VecInit(hitVec).asUInt)}\n")
451
452    VecInit(hitVecT).suggestName(s"sp_hitVecT")
453    VecInit(hitVec).suggestName(s"sp_hitVec")
454
455    (RegEnable(hit, stageDelay(1).fire),
456     RegEnable(hitData, stageDelay(1).fire),
457     RegEnable(hitData.prefetch, stageDelay(1).fire),
458     RegEnable(hitData.v, stageDelay(1).fire))
459  }
460  val spHitPerm = spHitData.perm.getOrElse(0.U.asTypeOf(new PtePermBundle))
461  val spHitLevel = spHitData.level.getOrElse(0.U)
462
463  val check_res = Wire(new PageCacheRespBundle)
464  check_res.l1.apply(l1Hit, l1Pre, l1HitPPN)
465  check_res.l2.apply(l2Hit, l2Pre, l2HitPPN, ecc = l2eccError)
466  check_res.l3.apply(l3Hit, l3Pre, l3HitPPN, l3HitPerm, l3eccError, valid = l3HitValid, accessFault = l3HitAf)
467  check_res.sp.apply(spHit, spPre, spHitData.ppn, spHitPerm, false.B, spHitLevel, spValid)
468
469  val resp_res = Reg(new PageCacheRespBundle)
470  when (stageCheck(1).fire) { resp_res := check_res }
471
472  // stageResp bypass
473  val bypassed = Wire(Vec(3, Bool()))
474  bypassed.indices.foreach(i =>
475    bypassed(i) := stageResp.bits.bypassed(i) ||
476      ValidHoldBypass(refill_bypass(stageResp.bits.req_info.vpn, i, stageResp.bits.req_info.s2xlate),
477        OneCycleValid(stageCheck(1).fire, false.B) || io.refill.valid)
478  )
479
480  // stageResp bypass to hptw
481  val hptw_bypassed = Wire(Vec(3, Bool()))
482  hptw_bypassed.indices.foreach(i =>
483    hptw_bypassed(i) := stageResp.bits.bypassed(i) ||
484      ValidHoldBypass(refill_bypass(stageResp.bits.req_info.vpn, i, stageResp.bits.req_info.s2xlate),
485        io.resp.fire)
486  )
487
488  val isAllStage = stageResp.bits.req_info.s2xlate === allStage
489  val isOnlyStage2 = stageResp.bits.req_info.s2xlate === onlyStage2
490  val stage1Hit = (resp_res.l3.hit || resp_res.sp.hit) && isAllStage
491  val idx = stageResp.bits.req_info.vpn(2, 0)
492  val stage1Pf = !Mux(resp_res.l3.hit, resp_res.l3.v(idx), resp_res.sp.v)
493  io.resp.bits.req_info   := stageResp.bits.req_info
494  io.resp.bits.isFirst  := stageResp.bits.isFirst
495  io.resp.bits.hit      := (resp_res.l3.hit || resp_res.sp.hit) && (!isAllStage || isAllStage && stage1Pf)
496  io.resp.bits.bypassed := (bypassed(2) || (bypassed(1) && !resp_res.l2.hit) || (bypassed(0) && !resp_res.l1.hit)) && !isAllStage
497  io.resp.bits.prefetch := resp_res.l3.pre && resp_res.l3.hit || resp_res.sp.pre && resp_res.sp.hit
498  io.resp.bits.toFsm.l1Hit := resp_res.l1.hit && !stage1Hit && !isOnlyStage2 && !stageResp.bits.isHptwReq
499  io.resp.bits.toFsm.l2Hit := resp_res.l2.hit && !stage1Hit && !isOnlyStage2 && !stageResp.bits.isHptwReq
500  io.resp.bits.toFsm.ppn   := Mux(resp_res.l2.hit, resp_res.l2.ppn, resp_res.l1.ppn)
501  io.resp.bits.toFsm.stage1Hit := stage1Hit
502
503  io.resp.bits.isHptwReq := stageResp.bits.isHptwReq
504  io.resp.bits.toHptw.bypassed := (hptw_bypassed(2) || (hptw_bypassed(1) && !resp_res.l2.hit) || (hptw_bypassed(0) && !resp_res.l1.hit)) && stageResp.bits.isHptwReq
505  io.resp.bits.toHptw.id := stageResp.bits.hptwId
506  io.resp.bits.toHptw.l1Hit := resp_res.l1.hit && stageResp.bits.isHptwReq
507  io.resp.bits.toHptw.l2Hit := resp_res.l2.hit && stageResp.bits.isHptwReq
508  io.resp.bits.toHptw.ppn := Mux(resp_res.l2.hit, resp_res.l2.ppn, resp_res.l1.ppn)(ppnLen - 1, 0)
509  io.resp.bits.toHptw.resp.entry.tag := stageResp.bits.req_info.vpn
510  io.resp.bits.toHptw.resp.entry.asid := DontCare
511  io.resp.bits.toHptw.resp.entry.vmid.map(_ := io.csr_dup(0).hgatp.asid)
512  io.resp.bits.toHptw.resp.entry.level.map(_ := Mux(resp_res.l3.hit, 2.U, resp_res.sp.level))
513  io.resp.bits.toHptw.resp.entry.prefetch := from_pre(stageResp.bits.req_info.source)
514  io.resp.bits.toHptw.resp.entry.ppn := Mux(resp_res.l3.hit, resp_res.l3.ppn(idx), resp_res.sp.ppn)(ppnLen - 1, 0)
515  io.resp.bits.toHptw.resp.entry.perm.map(_ := Mux(resp_res.l3.hit, resp_res.l3.perm(idx), resp_res.sp.perm))
516  io.resp.bits.toHptw.resp.entry.v := Mux(resp_res.l3.hit, resp_res.l3.v(idx), resp_res.sp.v)
517  io.resp.bits.toHptw.resp.gpf := !io.resp.bits.toHptw.resp.entry.v
518  io.resp.bits.toHptw.resp.gaf := Mux(resp_res.l3.hit, resp_res.l3.af(idx), false.B)
519
520  io.resp.bits.stage1.entry.map(_.tag := stageResp.bits.req_info.vpn(vpnLen - 1, 3))
521  io.resp.bits.stage1.entry.map(_.asid := Mux(stageResp.bits.req_info.hasS2xlate(), io.csr_dup(0).vsatp.asid, io.csr_dup(0).satp.asid)) // DontCare
522  io.resp.bits.stage1.entry.map(_.vmid.map(_ := io.csr_dup(0).hgatp.asid))
523  io.resp.bits.stage1.entry.map(_.level.map(_ := Mux(resp_res.l3.hit, 2.U, Mux(resp_res.sp.hit, resp_res.sp.level, Mux(resp_res.l2.hit, 1.U, 0.U))))) // leaf page is first
524  io.resp.bits.stage1.entry.map(_.prefetch := from_pre(stageResp.bits.req_info.source))
525  for (i <- 0 until tlbcontiguous) {
526    io.resp.bits.stage1.entry(i).ppn := Mux(resp_res.l3.hit, resp_res.l3.ppn(i)(gvpnLen - 1, sectortlbwidth), Mux(resp_res.sp.hit, resp_res.sp.ppn(gvpnLen - 1, sectortlbwidth), Mux(resp_res.l2.hit, resp_res.l2.ppn(gvpnLen - 1, sectortlbwidth), resp_res.l1.ppn(gvpnLen - 1, sectortlbwidth))))
527    io.resp.bits.stage1.entry(i).ppn_low := Mux(resp_res.l3.hit, resp_res.l3.ppn(i)(sectortlbwidth - 1, 0), Mux(resp_res.sp.hit, resp_res.sp.ppn(sectortlbwidth - 1, 0), Mux(resp_res.l2.hit, resp_res.l2.ppn(sectortlbwidth - 1, 0), resp_res.l1.ppn(sectortlbwidth - 1, 0))))
528    io.resp.bits.stage1.entry(i).perm.map(_ := Mux(resp_res.l3.hit, resp_res.l3.perm(i), resp_res.sp.perm))
529    io.resp.bits.stage1.entry(i).v := Mux(resp_res.l3.hit, resp_res.l3.v(i), Mux(resp_res.sp.hit, resp_res.sp.v, Mux(resp_res.l2.hit, resp_res.l2.v, resp_res.l1.v)))
530    io.resp.bits.stage1.entry(i).pf := !io.resp.bits.stage1.entry(i).v
531    io.resp.bits.stage1.entry(i).af := Mux(resp_res.l3.hit, resp_res.l3.af(i), false.B)
532  }
533  io.resp.bits.stage1.pteidx := UIntToOH(idx).asBools
534  io.resp.bits.stage1.not_super := Mux(resp_res.l3.hit, true.B, false.B)
535  io.resp.valid := stageResp.valid
536  XSError(stageResp.valid && resp_res.l3.hit && resp_res.sp.hit, "normal page and super page both hit")
537  XSError(stageResp.valid && io.resp.bits.hit && bypassed(2), "page cache, bypassed but hit")
538
539  // refill Perf
540  val l1RefillPerf = Wire(Vec(l2tlbParams.l1Size, Bool()))
541  val l2RefillPerf = Wire(Vec(l2tlbParams.l2nWays, Bool()))
542  val l3RefillPerf = Wire(Vec(l2tlbParams.l3nWays, Bool()))
543  val spRefillPerf = Wire(Vec(l2tlbParams.spSize, Bool()))
544  l1RefillPerf.map(_ := false.B)
545  l2RefillPerf.map(_ := false.B)
546  l3RefillPerf.map(_ := false.B)
547  spRefillPerf.map(_ := false.B)
548
549  // refill
550  l2.io.w.req <> DontCare
551  l3.io.w.req <> DontCare
552  l2.io.w.req.valid := false.B
553  l3.io.w.req.valid := false.B
554
555  val memRdata = refill.ptes
556  val memPtes = (0 until (l2tlbParams.blockBytes/(XLEN/8))).map(i => memRdata((i+1)*XLEN-1, i*XLEN).asTypeOf(new PteBundle))
557  val memSelData = io.refill.bits.sel_pte_dup
558  val memPte = memSelData.map(a => a.asTypeOf(new PteBundle))
559
560  // TODO: handle sfenceLatch outsize
561  when (!flush_dup(0) && refill.levelOH.l1 && !memPte(0).isLeaf() && !memPte(0).isPf(refill.level_dup(0)) && Mux(refill.req_info_dup(0).s2xlate === allStage, true.B, !memPte(0).isAf())) {
562    // val refillIdx = LFSR64()(log2Up(l2tlbParams.l1Size)-1,0) // TODO: may be LRU
563    val refillIdx = replaceWrapper(l1v, ptwl1replace.way)
564    refillIdx.suggestName(s"PtwL1RefillIdx")
565    val rfOH = UIntToOH(refillIdx)
566    l1(refillIdx).refill(
567      refill.req_info_dup(0).vpn,
568      Mux(refill.req_info_dup(0).s2xlate =/= noS2xlate, io.csr_dup(0).vsatp.asid, io.csr_dup(0).satp.asid),
569      io.csr_dup(0).hgatp.asid,
570      memSelData(0),
571      0.U,
572      refill_prefetch_dup(0)
573    )
574    ptwl1replace.access(refillIdx)
575    l1v := l1v | rfOH
576    l1g := (l1g & ~rfOH) | Mux(memPte(0).perm.g, rfOH, 0.U)
577    l1h(refillIdx) := refill_h(0)
578
579    for (i <- 0 until l2tlbParams.l1Size) {
580      l1RefillPerf(i) := i.U === refillIdx
581    }
582
583    XSDebug(p"[l1 refill] refillIdx:${refillIdx} refillEntry:${l1(refillIdx).genPtwEntry(refill.req_info_dup(0).vpn, Mux(refill.req_info_dup(0).s2xlate =/= noS2xlate, io.csr_dup(0).vsatp.asid, io.csr_dup(0).satp.asid), memSelData(0), 0.U, prefetch = refill_prefetch_dup(0))}\n")
584    XSDebug(p"[l1 refill] l1v:${Binary(l1v)}->${Binary(l1v | rfOH)} l1g:${Binary(l1g)}->${Binary((l1g & ~rfOH) | Mux(memPte(0).perm.g, rfOH, 0.U))}\n")
585
586    refillIdx.suggestName(s"l1_refillIdx")
587    rfOH.suggestName(s"l1_rfOH")
588  }
589
590  when (!flush_dup(1) && refill.levelOH.l2 && !memPte(1).isLeaf() && !memPte(1).isPf(refill.level_dup(1)) && Mux(refill.req_info_dup(1).s2xlate === allStage, true.B, !memPte(1).isAf())) {
591    val refillIdx = genPtwL2SetIdx(refill.req_info_dup(1).vpn)
592    val victimWay = replaceWrapper(getl2vSet(refill.req_info_dup(1).vpn), ptwl2replace.way(refillIdx))
593    val victimWayOH = UIntToOH(victimWay)
594    val rfvOH = UIntToOH(Cat(refillIdx, victimWay))
595    val wdata = Wire(l2EntryType)
596    wdata.gen(
597      vpn = refill.req_info_dup(1).vpn,
598      asid = Mux(refill.req_info_dup(1).s2xlate =/= noS2xlate, io.csr_dup(1).vsatp.asid, io.csr_dup(1).satp.asid),
599      vmid = io.csr_dup(1).hgatp.asid,
600      data = memRdata,
601      levelUInt = 1.U,
602      refill_prefetch_dup(1),
603      refill.req_info_dup(1).s2xlate
604    )
605    l2.io.w.apply(
606      valid = true.B,
607      setIdx = refillIdx,
608      data = wdata,
609      waymask = victimWayOH
610    )
611    ptwl2replace.access(refillIdx, victimWay)
612    l2v := l2v | rfvOH
613    l2g := l2g & ~rfvOH | Mux(Cat(memPtes.map(_.perm.g)).andR, rfvOH, 0.U)
614    l2h(refillIdx)(victimWay) := refill_h(1)
615
616    for (i <- 0 until l2tlbParams.l2nWays) {
617      l2RefillPerf(i) := i.U === victimWay
618    }
619
620    XSDebug(p"[l2 refill] refillIdx:0x${Hexadecimal(refillIdx)} victimWay:${victimWay} victimWayOH:${Binary(victimWayOH)} rfvOH(in UInt):${Cat(refillIdx, victimWay)}\n")
621    XSDebug(p"[l2 refill] refilldata:0x${wdata}\n")
622    XSDebug(p"[l2 refill] l2v:${Binary(l2v)} -> ${Binary(l2v | rfvOH)}\n")
623    XSDebug(p"[l2 refill] l2g:${Binary(l2g)} -> ${Binary(l2g & ~rfvOH | Mux(Cat(memPtes.map(_.perm.g)).andR, rfvOH, 0.U))}\n")
624
625    refillIdx.suggestName(s"l2_refillIdx")
626    victimWay.suggestName(s"l2_victimWay")
627    victimWayOH.suggestName(s"l2_victimWayOH")
628    rfvOH.suggestName(s"l2_rfvOH")
629  }
630
631  when (!flush_dup(2) && refill.levelOH.l3) {
632    val refillIdx = genPtwL3SetIdx(refill.req_info_dup(2).vpn)
633    val victimWay = replaceWrapper(getl3vSet(refill.req_info_dup(2).vpn), ptwl3replace.way(refillIdx))
634    val victimWayOH = UIntToOH(victimWay)
635    val rfvOH = UIntToOH(Cat(refillIdx, victimWay))
636    val wdata = Wire(l3EntryType)
637    wdata.gen(
638      vpn =  refill.req_info_dup(2).vpn,
639      asid = Mux(refill.req_info_dup(2).s2xlate =/= noS2xlate, io.csr_dup(2).vsatp.asid, io.csr_dup(2).satp.asid),
640      vmid = io.csr_dup(2).hgatp.asid,
641      data = memRdata,
642      levelUInt = 2.U,
643      refill_prefetch_dup(2),
644      refill.req_info_dup(2).s2xlate
645    )
646    l3.io.w.apply(
647      valid = true.B,
648      setIdx = refillIdx,
649      data = wdata,
650      waymask = victimWayOH
651    )
652    ptwl3replace.access(refillIdx, victimWay)
653    l3v := l3v | rfvOH
654    l3g := l3g & ~rfvOH | Mux(Cat(memPtes.map(_.perm.g)).andR, rfvOH, 0.U)
655    l3h(refillIdx)(victimWay) := refill_h(2)
656
657    for (i <- 0 until l2tlbParams.l3nWays) {
658      l3RefillPerf(i) := i.U === victimWay
659    }
660
661    XSDebug(p"[l3 refill] refillIdx:0x${Hexadecimal(refillIdx)} victimWay:${victimWay} victimWayOH:${Binary(victimWayOH)} rfvOH(in UInt):${Cat(refillIdx, victimWay)}\n")
662    XSDebug(p"[l3 refill] refilldata:0x${wdata}\n")
663    XSDebug(p"[l3 refill] l3v:${Binary(l3v)} -> ${Binary(l3v | rfvOH)}\n")
664    XSDebug(p"[l3 refill] l3g:${Binary(l3g)} -> ${Binary(l3g & ~rfvOH | Mux(Cat(memPtes.map(_.perm.g)).andR, rfvOH, 0.U))}\n")
665
666    refillIdx.suggestName(s"l3_refillIdx")
667    victimWay.suggestName(s"l3_victimWay")
668    victimWayOH.suggestName(s"l3_victimWayOH")
669    rfvOH.suggestName(s"l3_rfvOH")
670  }
671
672
673  // misc entries: super & invalid
674  when (!flush_dup(0) && refill.levelOH.sp && (memPte(0).isLeaf() || memPte(0).isPf(refill.level_dup(0))) && Mux(refill.req_info_dup(0).s2xlate === allStage, true.B, !memPte(0).isAf())) {
675    val refillIdx = spreplace.way// LFSR64()(log2Up(l2tlbParams.spSize)-1,0) // TODO: may be LRU
676    val rfOH = UIntToOH(refillIdx)
677    sp(refillIdx).refill(
678      refill.req_info_dup(0).vpn,
679      Mux(refill.req_info_dup(0).s2xlate =/= noS2xlate, io.csr_dup(0).vsatp.asid, io.csr_dup(0).satp.asid),
680      io.csr_dup(0).hgatp.asid,
681      memSelData(0),
682      refill.level_dup(2),
683      refill_prefetch_dup(0),
684      !memPte(0).isPf(refill.level_dup(0)),
685    )
686    spreplace.access(refillIdx)
687    spv := spv | rfOH
688    spg := spg & ~rfOH | Mux(memPte(0).perm.g, rfOH, 0.U)
689    sph(refillIdx) := refill_h(0)
690
691    for (i <- 0 until l2tlbParams.spSize) {
692      spRefillPerf(i) := i.U === refillIdx
693    }
694
695    XSDebug(p"[sp refill] refillIdx:${refillIdx} refillEntry:${sp(refillIdx).genPtwEntry(refill.req_info_dup(0).vpn, Mux(refill.req_info_dup(0).s2xlate =/= noS2xlate, io.csr_dup(0).vsatp.asid, io.csr_dup(0).satp.asid), memSelData(0), refill.level_dup(0), refill_prefetch_dup(0))}\n")
696    XSDebug(p"[sp refill] spv:${Binary(spv)}->${Binary(spv | rfOH)} spg:${Binary(spg)}->${Binary(spg & ~rfOH | Mux(memPte(0).perm.g, rfOH, 0.U))}\n")
697
698    refillIdx.suggestName(s"sp_refillIdx")
699    rfOH.suggestName(s"sp_rfOH")
700  }
701
702  val l2eccFlush = resp_res.l2.ecc && stageResp_valid_1cycle_dup(0) // RegNext(l2eccError, init = false.B)
703  val l3eccFlush = resp_res.l3.ecc && stageResp_valid_1cycle_dup(1) // RegNext(l3eccError, init = false.B)
704  val eccVpn = stageResp.bits.req_info.vpn
705
706  XSError(l2eccFlush, "l2tlb.cache.l2 ecc error. Should not happen at sim stage")
707  XSError(l3eccFlush, "l2tlb.cache.l3 ecc error. Should not happen at sim stage")
708  when (l2eccFlush) {
709    val flushSetIdxOH = UIntToOH(genPtwL2SetIdx(eccVpn))
710    val flushMask = VecInit(flushSetIdxOH.asBools.map { a => Fill(l2tlbParams.l2nWays, a.asUInt) }).asUInt
711    l2v := l2v & ~flushMask
712    l2g := l2g & ~flushMask
713  }
714
715  when (l3eccFlush) {
716    val flushSetIdxOH = UIntToOH(genPtwL3SetIdx(eccVpn))
717    val flushMask = VecInit(flushSetIdxOH.asBools.map { a => Fill(l2tlbParams.l3nWays, a.asUInt) }).asUInt
718    l3v := l3v & ~flushMask
719    l3g := l3g & ~flushMask
720  }
721
722  // sfence for l3
723  val sfence_valid_l3 = sfence_dup(3).valid && !sfence_dup(3).bits.hg && !sfence_dup(3).bits.hv
724  when (sfence_valid_l3) {
725    val l3hhit = VecInit(l3h.flatMap(_.map{a => io.csr_dup(0).priv.virt && a === onlyStage1 || !io.csr_dup(0).priv.virt && a === noS2xlate})).asUInt
726    val sfence_vpn = sfence_dup(3).bits.addr(sfence_dup(3).bits.addr.getWidth-1, offLen)
727    when (sfence_dup(3).bits.rs1/*va*/) {
728      when (sfence_dup(3).bits.rs2) {
729        // all va && all asid
730        l3v := l3v & ~l3hhit
731      } .otherwise {
732        // all va && specific asid except global
733        l3v := l3v & (l3g | ~l3hhit)
734      }
735    } .otherwise {
736      // val flushMask = UIntToOH(genTlbL2Idx(sfence.bits.addr(sfence.bits.addr.getWidth-1, offLen)))
737      val flushSetIdxOH = UIntToOH(genPtwL3SetIdx(sfence_vpn))
738      // val flushMask = VecInit(flushSetIdxOH.asBools.map(Fill(l2tlbParams.l3nWays, _.asUInt))).asUInt
739      val flushMask = VecInit(flushSetIdxOH.asBools.map { a => Fill(l2tlbParams.l3nWays, a.asUInt) }).asUInt
740      flushSetIdxOH.suggestName(s"sfence_nrs1_flushSetIdxOH")
741      flushMask.suggestName(s"sfence_nrs1_flushMask")
742
743      when (sfence_dup(3).bits.rs2) {
744        // specific leaf of addr && all asid
745        l3v := l3v & ~flushMask & ~l3hhit
746      } .otherwise {
747        // specific leaf of addr && specific asid
748        l3v := l3v & (~flushMask | l3g | ~l3hhit)
749      }
750    }
751  }
752
753  // hfencev, simple implementation for l3
754  val hfencev_valid_l3 = sfence_dup(3).valid && sfence_dup(3).bits.hv
755  when(hfencev_valid_l3) {
756    val flushMask = VecInit(l3h.flatMap(_.map(_  === onlyStage1))).asUInt
757    l3v := l3v & ~flushMask // all VS-stage l3 pte
758  }
759
760  // hfenceg, simple implementation for l3
761  val hfenceg_valid_l3 = sfence_dup(3).valid && sfence_dup(3).bits.hg
762  when(hfenceg_valid_l3) {
763    val flushMask = VecInit(l3h.flatMap(_.map(_ === onlyStage2))).asUInt
764    l3v := l3v & ~flushMask // all G-stage l3 pte
765  }
766
767
768  val l1asidhit = VecInit(l1asids.map(_ === sfence_dup(0).bits.id)).asUInt
769  val spasidhit = VecInit(spasids.map(_ === sfence_dup(0).bits.id)).asUInt
770  val sfence_valid = sfence_dup(0).valid && !sfence_dup(0).bits.hg && !sfence_dup(0).bits.hv
771  when (sfence_valid) {
772    val l1vmidhit = VecInit(l1vmids.map(_.getOrElse(0.U) === io.csr_dup(0).hgatp.asid)).asUInt
773    val spvmidhit = VecInit(spvmids.map(_.getOrElse(0.U) === io.csr_dup(0).hgatp.asid)).asUInt
774    val l1hhit = VecInit(l1h.map{a => io.csr_dup(0).priv.virt && a === onlyStage1 || !io.csr_dup(0).priv.virt && a === noS2xlate}).asUInt
775    val sphhit = VecInit(sph.map{a => io.csr_dup(0).priv.virt && a === onlyStage1 || !io.csr_dup(0).priv.virt && a === noS2xlate}).asUInt
776    val l2hhit = VecInit(l2h.flatMap(_.map{a => io.csr_dup(0).priv.virt && a === onlyStage1 || !io.csr_dup(0).priv.virt && a === noS2xlate})).asUInt
777    val sfence_vpn = sfence_dup(0).bits.addr(sfence_dup(0).bits.addr.getWidth-1, offLen)
778
779    when (sfence_dup(0).bits.rs1/*va*/) {
780      when (sfence_dup(0).bits.rs2) {
781        // all va && all asid
782        l2v := l2v & ~l2hhit
783        l1v := l1v & ~(l1hhit & VecInit(l1vmidhit.asBools.map{a => io.csr_dup(0).priv.virt && a || !io.csr_dup(0).priv.virt}).asUInt)
784        spv := spv & ~(sphhit & VecInit(spvmidhit.asBools.map{a => io.csr_dup(0).priv.virt && a || !io.csr_dup(0).priv.virt}).asUInt)
785      } .otherwise {
786        // all va && specific asid except global
787        l2v := l2v & (l2g | ~l2hhit)
788        l1v := l1v & ~(~l1g & l1hhit & l1asidhit & VecInit(l1vmidhit.asBools.map{a => io.csr_dup(0).priv.virt && a || !io.csr_dup(0).priv.virt}).asUInt)
789        spv := spv & ~(~spg & sphhit & spasidhit & VecInit(spvmidhit.asBools.map{a => io.csr_dup(0).priv.virt && a || !io.csr_dup(0).priv.virt}).asUInt)
790      }
791    } .otherwise {
792      when (sfence_dup(0).bits.rs2) {
793        // specific leaf of addr && all asid
794        spv := spv & ~(sphhit & VecInit(sp.map(_.hit(sfence_vpn, sfence_dup(0).bits.id, sfence_dup(0).bits.id, io.csr_dup(0).hgatp.asid, ignoreAsid = true, s2xlate = io.csr_dup(0).priv.virt))).asUInt)
795      } .otherwise {
796        // specific leaf of addr && specific asid
797        spv := spv & ~(~spg & sphhit & VecInit(sp.map(_.hit(sfence_vpn, sfence_dup(0).bits.id, sfence_dup(0).bits.id, io.csr_dup(0).hgatp.asid, s2xlate = io.csr_dup(0).priv.virt))).asUInt)
798      }
799    }
800  }
801
802  val hfencev_valid = sfence_dup(0).valid && sfence_dup(0).bits.hv
803  when (hfencev_valid) {
804    val l1vmidhit = VecInit(l1vmids.map(_.getOrElse(0.U) === io.csr_dup(0).hgatp.asid)).asUInt
805    val spvmidhit = VecInit(spvmids.map(_.getOrElse(0.U) === io.csr_dup(0).hgatp.asid)).asUInt
806    val l1hhit = VecInit(l1h.map(_ === onlyStage1)).asUInt
807    val sphhit = VecInit(sph.map(_ === onlyStage1)).asUInt
808    val l2hhit = VecInit(l2h.flatMap(_.map(_ === onlyStage1))).asUInt
809    val hfencev_vpn = sfence_dup(0).bits.addr(sfence_dup(0).bits.addr.getWidth-1, offLen)
810    when(sfence_dup(0).bits.rs1) {
811      when(sfence_dup(0).bits.rs2) {
812        l2v := l2v & ~l2hhit
813        l1v := l1v & ~(l1hhit & l1vmidhit)
814        spv := spv & ~(sphhit & spvmidhit)
815      }.otherwise {
816        l2v := l2v & (l2g | ~l2hhit)
817        l1v := l1v & ~(~l1g & l1hhit & l1asidhit & l1vmidhit)
818        spv := spv & ~(~spg & sphhit & spasidhit & spvmidhit)
819      }
820    }.otherwise {
821      when(sfence_dup(0).bits.rs2) {
822        spv := spv & ~(sphhit & VecInit(sp.map(_.hit(hfencev_vpn, sfence_dup(0).bits.id, sfence_dup(0).bits.id, io.csr_dup(0).hgatp.asid, ignoreAsid = true, s2xlate = true.B))).asUInt)
823      }.otherwise {
824        spv := spv & ~(~spg & sphhit & VecInit(sp.map(_.hit(hfencev_vpn, sfence_dup(0).bits.id, sfence_dup(0).bits.id, io.csr_dup(0).hgatp.asid, s2xlate = true.B))).asUInt)
825      }
826    }
827  }
828
829
830  val hfenceg_valid = sfence_dup(0).valid && sfence_dup(0).bits.hg
831  when(hfenceg_valid) {
832    val l1vmidhit = VecInit(l1vmids.map(_.getOrElse(0.U) === sfence_dup(0).bits.id)).asUInt
833    val spvmidhit = VecInit(spvmids.map(_.getOrElse(0.U) === sfence_dup(0).bits.id)).asUInt
834    val l1hhit = VecInit(l1h.map(_ === onlyStage2)).asUInt
835    val sphhit = VecInit(sph.map(_ === onlyStage2)).asUInt
836    val l2hhit = VecInit(l2h.flatMap(_.map(_ === onlyStage2))).asUInt
837    val hfenceg_gvpn = (sfence_dup(0).bits.addr << 2)(sfence_dup(0).bits.addr.getWidth - 1, offLen)
838    when(sfence_dup(0).bits.rs1) {
839      when(sfence_dup(0).bits.rs2) {
840        l2v := l2v & ~l2hhit
841        l1v := l1v & ~l1hhit
842        spv := spv & ~sphhit
843      }.otherwise {
844        l2v := l2v & ~l2hhit
845        l1v := l1v & ~(l1hhit & l1vmidhit)
846        spv := spv & ~(sphhit & spvmidhit)
847      }
848    }.otherwise {
849      when(sfence_dup(0).bits.rs2) {
850        spv := spv & ~(sphhit & VecInit(sp.map(_.hit(hfenceg_gvpn, 0.U, 0.U, sfence_dup(0).bits.id, ignoreAsid = true, s2xlate = false.B))).asUInt)
851      }.otherwise {
852        spv := spv & ~(~spg & sphhit & VecInit(sp.map(_.hit(hfenceg_gvpn, 0.U, 0.U, sfence_dup(0).bits.id, ignoreAsid = true, s2xlate = true.B))).asUInt)
853      }
854    }
855  }
856
857  def InsideStageConnect(in: DecoupledIO[PtwCacheReq], out: DecoupledIO[PtwCacheReq], inFire: Bool): Unit = {
858    in.ready := !in.valid || out.ready
859    out.valid := in.valid
860    out.bits := in.bits
861    out.bits.bypassed.zip(in.bits.bypassed).zipWithIndex.map{ case (b, i) =>
862      val bypassed_reg = Reg(Bool())
863      val bypassed_wire = refill_bypass(in.bits.req_info.vpn, i, in.bits.req_info.s2xlate) && io.refill.valid
864      when (inFire) { bypassed_reg := bypassed_wire }
865      .elsewhen (io.refill.valid) { bypassed_reg := bypassed_reg || bypassed_wire }
866
867      b._1 := b._2 || (bypassed_wire || (bypassed_reg && !inFire))
868    }
869  }
870
871  // Perf Count
872  val resp_l3 = resp_res.l3.hit
873  val resp_sp = resp_res.sp.hit
874  val resp_l1_pre = resp_res.l1.pre
875  val resp_l2_pre = resp_res.l2.pre
876  val resp_l3_pre = resp_res.l3.pre
877  val resp_sp_pre = resp_res.sp.pre
878  val base_valid_access_0 = !from_pre(io.resp.bits.req_info.source) && io.resp.fire
879  XSPerfAccumulate("access", base_valid_access_0)
880  XSPerfAccumulate("l1_hit", base_valid_access_0 && io.resp.bits.toFsm.l1Hit && !io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit)
881  XSPerfAccumulate("l2_hit", base_valid_access_0 && io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit)
882  XSPerfAccumulate("l3_hit", base_valid_access_0 && resp_l3)
883  XSPerfAccumulate("sp_hit", base_valid_access_0 && resp_sp)
884  XSPerfAccumulate("pte_hit",base_valid_access_0 && io.resp.bits.hit)
885
886  XSPerfAccumulate("l1_hit_pre", base_valid_access_0 && resp_l1_pre && io.resp.bits.toFsm.l1Hit && !io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit)
887  XSPerfAccumulate("l2_hit_pre", base_valid_access_0 && resp_l2_pre && io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit)
888  XSPerfAccumulate("l3_hit_pre", base_valid_access_0 && resp_l3_pre && resp_l3)
889  XSPerfAccumulate("sp_hit_pre", base_valid_access_0 && resp_sp_pre && resp_sp)
890  XSPerfAccumulate("pte_hit_pre",base_valid_access_0 && (resp_l3_pre && resp_l3 || resp_sp_pre && resp_sp) && io.resp.bits.hit)
891
892  val base_valid_access_1 = from_pre(io.resp.bits.req_info.source) && io.resp.fire
893  XSPerfAccumulate("pre_access", base_valid_access_1)
894  XSPerfAccumulate("pre_l1_hit", base_valid_access_1 && io.resp.bits.toFsm.l1Hit && !io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit)
895  XSPerfAccumulate("pre_l2_hit", base_valid_access_1 && io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit)
896  XSPerfAccumulate("pre_l3_hit", base_valid_access_1 && resp_l3)
897  XSPerfAccumulate("pre_sp_hit", base_valid_access_1 && resp_sp)
898  XSPerfAccumulate("pre_pte_hit",base_valid_access_1 && io.resp.bits.hit)
899
900  XSPerfAccumulate("pre_l1_hit_pre", base_valid_access_1 && resp_l1_pre && io.resp.bits.toFsm.l1Hit && !io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit)
901  XSPerfAccumulate("pre_l2_hit_pre", base_valid_access_1 && resp_l2_pre && io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit)
902  XSPerfAccumulate("pre_l3_hit_pre", base_valid_access_1 && resp_l3_pre && resp_l3)
903  XSPerfAccumulate("pre_sp_hit_pre", base_valid_access_1 && resp_sp_pre && resp_sp)
904  XSPerfAccumulate("pre_pte_hit_pre",base_valid_access_1 && (resp_l3_pre && resp_l3 || resp_sp_pre && resp_sp) && io.resp.bits.hit)
905
906  val base_valid_access_2 = stageResp.bits.isFirst && !from_pre(io.resp.bits.req_info.source) && io.resp.fire
907  XSPerfAccumulate("access_first", base_valid_access_2)
908  XSPerfAccumulate("l1_hit_first", base_valid_access_2 && io.resp.bits.toFsm.l1Hit && !io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit)
909  XSPerfAccumulate("l2_hit_first", base_valid_access_2 && io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit)
910  XSPerfAccumulate("l3_hit_first", base_valid_access_2 && resp_l3)
911  XSPerfAccumulate("sp_hit_first", base_valid_access_2 && resp_sp)
912  XSPerfAccumulate("pte_hit_first",base_valid_access_2 && io.resp.bits.hit)
913
914  XSPerfAccumulate("l1_hit_pre_first", base_valid_access_2 && resp_l1_pre && io.resp.bits.toFsm.l1Hit && !io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit)
915  XSPerfAccumulate("l2_hit_pre_first", base_valid_access_2 && resp_l2_pre && io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit)
916  XSPerfAccumulate("l3_hit_pre_first", base_valid_access_2 && resp_l3_pre && resp_l3)
917  XSPerfAccumulate("sp_hit_pre_first", base_valid_access_2 && resp_sp_pre && resp_sp)
918  XSPerfAccumulate("pte_hit_pre_first",base_valid_access_2 && (resp_l3_pre && resp_l3 || resp_sp_pre && resp_sp) && io.resp.bits.hit)
919
920  val base_valid_access_3 = stageResp.bits.isFirst && from_pre(io.resp.bits.req_info.source) && io.resp.fire
921  XSPerfAccumulate("pre_access_first", base_valid_access_3)
922  XSPerfAccumulate("pre_l1_hit_first", base_valid_access_3 && io.resp.bits.toFsm.l1Hit && !io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit)
923  XSPerfAccumulate("pre_l2_hit_first", base_valid_access_3 && io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit)
924  XSPerfAccumulate("pre_l3_hit_first", base_valid_access_3 && resp_l3)
925  XSPerfAccumulate("pre_sp_hit_first", base_valid_access_3 && resp_sp)
926  XSPerfAccumulate("pre_pte_hit_first", base_valid_access_3 && io.resp.bits.hit)
927
928  XSPerfAccumulate("pre_l1_hit_pre_first", base_valid_access_3 && resp_l1_pre && io.resp.bits.toFsm.l1Hit && !io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit)
929  XSPerfAccumulate("pre_l2_hit_pre_first", base_valid_access_3 && resp_l2_pre && io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit)
930  XSPerfAccumulate("pre_l3_hit_pre_first", base_valid_access_3 && resp_l3_pre && resp_l3)
931  XSPerfAccumulate("pre_sp_hit_pre_first", base_valid_access_3 && resp_sp_pre && resp_sp)
932  XSPerfAccumulate("pre_pte_hit_pre_first",base_valid_access_3 && (resp_l3_pre && resp_l3 || resp_sp_pre && resp_sp) && io.resp.bits.hit)
933
934  XSPerfAccumulate("rwHarzad", io.req.valid && !io.req.ready)
935  XSPerfAccumulate("out_blocked", io.resp.valid && !io.resp.ready)
936  l1AccessPerf.zipWithIndex.map{ case (l, i) => XSPerfAccumulate(s"L1AccessIndex${i}", l) }
937  l2AccessPerf.zipWithIndex.map{ case (l, i) => XSPerfAccumulate(s"L2AccessIndex${i}", l) }
938  l3AccessPerf.zipWithIndex.map{ case (l, i) => XSPerfAccumulate(s"L3AccessIndex${i}", l) }
939  spAccessPerf.zipWithIndex.map{ case (l, i) => XSPerfAccumulate(s"SPAccessIndex${i}", l) }
940  l1RefillPerf.zipWithIndex.map{ case (l, i) => XSPerfAccumulate(s"L1RefillIndex${i}", l) }
941  l2RefillPerf.zipWithIndex.map{ case (l, i) => XSPerfAccumulate(s"L2RefillIndex${i}", l) }
942  l3RefillPerf.zipWithIndex.map{ case (l, i) => XSPerfAccumulate(s"L3RefillIndex${i}", l) }
943  spRefillPerf.zipWithIndex.map{ case (l, i) => XSPerfAccumulate(s"SPRefillIndex${i}", l) }
944
945  XSPerfAccumulate("l1Refill", Cat(l1RefillPerf).orR)
946  XSPerfAccumulate("l2Refill", Cat(l2RefillPerf).orR)
947  XSPerfAccumulate("l3Refill", Cat(l3RefillPerf).orR)
948  XSPerfAccumulate("spRefill", Cat(spRefillPerf).orR)
949  XSPerfAccumulate("l1Refill_pre", Cat(l1RefillPerf).orR && refill_prefetch_dup(0))
950  XSPerfAccumulate("l2Refill_pre", Cat(l2RefillPerf).orR && refill_prefetch_dup(0))
951  XSPerfAccumulate("l3Refill_pre", Cat(l3RefillPerf).orR && refill_prefetch_dup(0))
952  XSPerfAccumulate("spRefill_pre", Cat(spRefillPerf).orR && refill_prefetch_dup(0))
953
954  // debug
955  XSDebug(sfence_dup(0).valid, p"[sfence] original v and g vector:\n")
956  XSDebug(sfence_dup(0).valid, p"[sfence] l1v:${Binary(l1v)}\n")
957  XSDebug(sfence_dup(0).valid, p"[sfence] l2v:${Binary(l2v)}\n")
958  XSDebug(sfence_dup(0).valid, p"[sfence] l3v:${Binary(l3v)}\n")
959  XSDebug(sfence_dup(0).valid, p"[sfence] l3g:${Binary(l3g)}\n")
960  XSDebug(sfence_dup(0).valid, p"[sfence] spv:${Binary(spv)}\n")
961  XSDebug(RegNext(sfence_dup(0).valid), p"[sfence] new v and g vector:\n")
962  XSDebug(RegNext(sfence_dup(0).valid), p"[sfence] l1v:${Binary(l1v)}\n")
963  XSDebug(RegNext(sfence_dup(0).valid), p"[sfence] l2v:${Binary(l2v)}\n")
964  XSDebug(RegNext(sfence_dup(0).valid), p"[sfence] l3v:${Binary(l3v)}\n")
965  XSDebug(RegNext(sfence_dup(0).valid), p"[sfence] l3g:${Binary(l3g)}\n")
966  XSDebug(RegNext(sfence_dup(0).valid), p"[sfence] spv:${Binary(spv)}\n")
967
968  val perfEvents = Seq(
969    ("access           ", base_valid_access_0             ),
970    ("l1_hit           ", l1Hit                           ),
971    ("l2_hit           ", l2Hit                           ),
972    ("l3_hit           ", l3Hit                           ),
973    ("sp_hit           ", spHit                           ),
974    ("pte_hit          ", l3Hit || spHit                  ),
975    ("rwHarzad         ",  io.req.valid && !io.req.ready  ),
976    ("out_blocked      ",  io.resp.valid && !io.resp.ready),
977  )
978  generatePerfEvent()
979}
980