xref: /XiangShan/src/main/scala/xiangshan/mem/MemCommon.scala (revision 519244c70fb0960c61014ad524e609af0918a818) 
1/***************************************************************************************
2* Copyright (c) 2024 Beijing Institute of Open Source Chip (BOSC)
3* Copyright (c) 2020-2024 Institute of Computing Technology, Chinese Academy of Sciences
4* Copyright (c) 2020-2021 Peng Cheng Laboratory
5*
6* XiangShan is licensed under Mulan PSL v2.
7* You can use this software according to the terms and conditions of the Mulan PSL v2.
8* You may obtain a copy of Mulan PSL v2 at:
9*          http://license.coscl.org.cn/MulanPSL2
10*
11* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
12* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
13* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
14*
15* See the Mulan PSL v2 for more details.
16***************************************************************************************/
17
18package xiangshan.mem
19
20
21import org.chipsalliance.cde.config.Parameters
22import chisel3._
23import chisel3.util._
24import utility._
25import utils._
26import xiangshan._
27import xiangshan.backend.Bundles.{DynInst, MemExuInput}
28import xiangshan.backend.rob.RobPtr
29import xiangshan.cache._
30import xiangshan.backend.fu.FenceToSbuffer
31import xiangshan.cache.wpu.ReplayCarry
32import xiangshan.mem.prefetch.PrefetchReqBundle
33import math._
34
35object genWmask {
36  def apply(addr: UInt, sizeEncode: UInt): UInt = {
37    (LookupTree(sizeEncode, List(
38      "b00".U -> 0x1.U, //0001 << addr(2:0)
39      "b01".U -> 0x3.U, //0011
40      "b10".U -> 0xf.U, //1111
41      "b11".U -> 0xff.U //11111111
42    )) << addr(2, 0)).asUInt
43  }
44}
45
46object genVWmask {
47  def apply(addr: UInt, sizeEncode: UInt): UInt = {
48    (LookupTree(sizeEncode, List(
49      "b00".U -> 0x1.U, //0001 << addr(2:0)
50      "b01".U -> 0x3.U, //0011
51      "b10".U -> 0xf.U, //1111
52      "b11".U -> 0xff.U //11111111
53    )) << addr(3, 0)).asUInt
54  }
55}
56
57object genBasemask {
58  /**
59   *
60   * @param addr
61   * @param sizeEncode
62   * @return Return 16-byte aligned mask.
63   *
64   *         Example:
65   *         Address: 0x80000003 Encoding size: ‘b11
66   *         Return: 0xff
67   */
68  def apply(addr: UInt, sizeEncode: UInt): UInt = {
69    LookupTree(sizeEncode, List(
70      "b00".U -> 0x1.U,
71      "b01".U -> 0x3.U,
72      "b10".U -> 0xf.U,
73      "b11".U -> 0xff.U
74    ))
75  }
76}
77
78object shiftDataToLow {
79  def apply(addr: UInt, data : UInt): UInt = {
80    Mux(addr(3), (data >> 64).asUInt, data)
81  }
82}
83object shiftMaskToLow {
84  def apply(addr: UInt, mask: UInt): UInt = {
85    Mux(addr(3), (mask >> 8).asUInt, mask)
86  }
87}
88object shiftDataToHigh {
89  def apply(addr: UInt, data : UInt): UInt = {
90    Mux(addr(3), (data << 64).asUInt, data)
91  }
92}
93object shiftMaskToHigh {
94  def apply(addr: UInt, mask: UInt): UInt = {
95    Mux(addr(3), (mask << 8).asUInt, mask)
96  }
97}
98
99class LsPipelineBundle(implicit p: Parameters) extends XSBundle
100  with HasDCacheParameters
101  with HasVLSUParameters {
102  val uop = new DynInst
103  val vaddr = UInt(VAddrBits.W)
104  // For exception vaddr generate
105  val fullva = UInt(XLEN.W)
106  val vaNeedExt = Bool()
107  val isHyper = Bool()
108  val paddr = UInt(PAddrBits.W)
109  val gpaddr = UInt(XLEN.W)
110  val isForVSnonLeafPTE = Bool()
111  // val func = UInt(6.W)
112  val mask = UInt((VLEN/8).W)
113  val data = UInt((VLEN+1).W)
114  val wlineflag = Bool() // store write the whole cache line
115
116  val miss = Bool()
117  val tlbMiss = Bool()
118  val ptwBack = Bool()
119  val af = Bool()
120  val nc = Bool()
121  val mmio = Bool()
122  val memBackTypeMM = Bool() // 1: main memory, 0: IO
123  val atomic = Bool()
124  val hasException = Bool()
125
126  val forwardMask = Vec(VLEN/8, Bool())
127  val forwardData = Vec(VLEN/8, UInt(8.W))
128
129  // prefetch
130  val isPrefetch = Bool()
131  val isHWPrefetch = Bool()
132  def isSWPrefetch = isPrefetch && !isHWPrefetch
133
134  // misalignBuffer
135  val isFrmMisAlignBuf = Bool()
136
137  // vector
138  val isvec = Bool()
139  val isLastElem = Bool()
140  val is128bit = Bool()
141  val uop_unit_stride_fof = Bool()
142  val usSecondInv = Bool()
143  val elemIdx = UInt(elemIdxBits.W)
144  val alignedType = UInt(alignTypeBits.W)
145  val mbIndex = UInt(max(vlmBindexBits, vsmBindexBits).W)
146  // val rob_idx_valid = Vec(2,Bool())
147  // val inner_idx = Vec(2,UInt(3.W))
148  // val rob_idx = Vec(2,new RobPtr)
149  val reg_offset = UInt(vOffsetBits.W)
150  val elemIdxInsideVd = UInt(elemIdxBits.W)
151  // val offset = Vec(2,UInt(4.W))
152  val vecActive = Bool() // 1: vector active element or scala mem operation, 0: vector not active element
153  val is_first_ele = Bool()
154  val vecBaseVaddr = UInt(VAddrBits.W)
155  val vecVaddrOffset = UInt(VAddrBits.W)
156  val vecTriggerMask = UInt((VLEN/8).W)
157  // val flowPtr = new VlflowPtr() // VLFlowQueue ptr
158  // val sflowPtr = new VsFlowPtr() // VSFlowQueue ptr
159
160  // For debug usage
161  val isFirstIssue = Bool()
162  val hasROBEntry = Bool()
163
164  // For load replay
165  val isLoadReplay = Bool()
166  val isFastPath = Bool()
167  val isFastReplay = Bool()
168  val replayCarry = new ReplayCarry(nWays)
169
170  // For dcache miss load
171  val mshrid = UInt(log2Up(cfg.nMissEntries).W)
172  val handledByMSHR = Bool()
173  val replacementUpdated = Bool()
174  val missDbUpdated = Bool()
175
176  val forward_tlDchannel = Bool()
177  val dcacheRequireReplay = Bool()
178  val delayedLoadError = Bool()
179  val lateKill = Bool()
180  val feedbacked = Bool()
181  val ldCancel = ValidUndirectioned(UInt(log2Ceil(LoadPipelineWidth).W))
182  // loadQueueReplay index.
183  val schedIndex = UInt(log2Up(LoadQueueReplaySize).W)
184  // hardware prefetch and fast replay no need to query tlb
185  val tlbNoQuery = Bool()
186
187  // misalign
188  val isMisalign          = Bool()
189  val isFinalSplit        = Bool()
190  val misalignWith16Byte  = Bool()
191  val misalignNeedWakeUp  = Bool()
192  val updateAddrValid     = Bool()
193}
194
195class LdPrefetchTrainBundle(implicit p: Parameters) extends LsPipelineBundle {
196  val meta_prefetch = UInt(L1PfSourceBits.W)
197  val meta_access = Bool()
198
199  def fromLsPipelineBundle(input: LsPipelineBundle, latch: Boolean = false, enable: Bool = true.B) = {
200    if (latch) vaddr := RegEnable(input.vaddr, enable) else vaddr := input.vaddr
201    if (latch) fullva := RegEnable(input.fullva, enable) else fullva := input.fullva
202    if (latch) vaNeedExt := RegEnable(input.vaNeedExt, enable) else vaNeedExt := input.vaNeedExt
203    if (latch) isHyper := RegEnable(input.isHyper, enable) else isHyper := input.isHyper
204    if (latch) paddr := RegEnable(input.paddr, enable) else paddr := input.paddr
205    if (latch) gpaddr := RegEnable(input.gpaddr, enable) else gpaddr := input.gpaddr
206    if (latch) isForVSnonLeafPTE := RegEnable(input.isForVSnonLeafPTE, enable) else isForVSnonLeafPTE := input.isForVSnonLeafPTE
207    if (latch) mask := RegEnable(input.mask, enable) else mask := input.mask
208    if (latch) data := RegEnable(input.data, enable) else data := input.data
209    if (latch) uop := RegEnable(input.uop, enable) else uop := input.uop
210    if (latch) wlineflag := RegEnable(input.wlineflag, enable) else wlineflag := input.wlineflag
211    if (latch) miss := RegEnable(input.miss, enable) else miss := input.miss
212    if (latch) tlbMiss := RegEnable(input.tlbMiss, enable) else tlbMiss := input.tlbMiss
213    if (latch) ptwBack := RegEnable(input.ptwBack, enable) else ptwBack := input.ptwBack
214    if (latch) af := RegEnable(input.af, enable) else af := input.af
215    if (latch) nc := RegEnable(input.nc, enable) else nc := input.nc
216    if (latch) mmio := RegEnable(input.mmio, enable) else mmio := input.mmio
217    if (latch) memBackTypeMM := RegEnable(input.memBackTypeMM, enable) else memBackTypeMM := input.memBackTypeMM
218    if (latch) forwardMask := RegEnable(input.forwardMask, enable) else forwardMask := input.forwardMask
219    if (latch) forwardData := RegEnable(input.forwardData, enable) else forwardData := input.forwardData
220    if (latch) isPrefetch := RegEnable(input.isPrefetch, enable) else isPrefetch := input.isPrefetch
221    if (latch) isHWPrefetch := RegEnable(input.isHWPrefetch, enable) else isHWPrefetch := input.isHWPrefetch
222    if (latch) isFrmMisAlignBuf := RegEnable(input.isFrmMisAlignBuf, enable) else isFrmMisAlignBuf := input.isFrmMisAlignBuf
223    if (latch) isFirstIssue := RegEnable(input.isFirstIssue, enable) else isFirstIssue := input.isFirstIssue
224    if (latch) hasROBEntry := RegEnable(input.hasROBEntry, enable) else hasROBEntry := input.hasROBEntry
225    if (latch) dcacheRequireReplay := RegEnable(input.dcacheRequireReplay, enable) else dcacheRequireReplay := input.dcacheRequireReplay
226    if (latch) schedIndex := RegEnable(input.schedIndex, enable) else schedIndex := input.schedIndex
227    if (latch) tlbNoQuery := RegEnable(input.tlbNoQuery, enable) else tlbNoQuery := input.tlbNoQuery
228    if (latch) isvec               := RegEnable(input.isvec, enable)               else isvec               := input.isvec
229    if (latch) isLastElem          := RegEnable(input.isLastElem, enable)          else isLastElem          := input.isLastElem
230    if (latch) is128bit            := RegEnable(input.is128bit, enable)            else is128bit            := input.is128bit
231    if (latch) vecActive           := RegEnable(input.vecActive, enable)           else vecActive           := input.vecActive
232    if (latch) is_first_ele        := RegEnable(input.is_first_ele, enable)        else is_first_ele        := input.is_first_ele
233    if (latch) uop_unit_stride_fof := RegEnable(input.uop_unit_stride_fof, enable) else uop_unit_stride_fof := input.uop_unit_stride_fof
234    if (latch) usSecondInv         := RegEnable(input.usSecondInv, enable)         else usSecondInv         := input.usSecondInv
235    if (latch) reg_offset          := RegEnable(input.reg_offset, enable)          else reg_offset          := input.reg_offset
236    if (latch) elemIdx             := RegEnable(input.elemIdx, enable)             else elemIdx             := input.elemIdx
237    if (latch) alignedType         := RegEnable(input.alignedType, enable)         else alignedType         := input.alignedType
238    if (latch) mbIndex             := RegEnable(input.mbIndex, enable)             else mbIndex             := input.mbIndex
239    if (latch) elemIdxInsideVd     := RegEnable(input.elemIdxInsideVd, enable)     else elemIdxInsideVd     := input.elemIdxInsideVd
240    if (latch) vecBaseVaddr        := RegEnable(input.vecBaseVaddr, enable)        else vecBaseVaddr        := input.vecBaseVaddr
241    if (latch) vecVaddrOffset      := RegEnable(input.vecVaddrOffset, enable)      else vecVaddrOffset      := input.vecVaddrOffset
242    if (latch) vecTriggerMask      := RegEnable(input.vecTriggerMask, enable)      else vecTriggerMask      := input.vecTriggerMask
243    // if (latch) flowPtr             := RegEnable(input.flowPtr, enable)             else flowPtr             := input.flowPtr
244    // if (latch) sflowPtr            := RegEnable(input.sflowPtr, enable)            else sflowPtr            := input.sflowPtr
245
246    meta_prefetch := DontCare
247    meta_access := DontCare
248    forward_tlDchannel := DontCare
249    mshrid := DontCare
250    replayCarry := DontCare
251    atomic := DontCare
252    isLoadReplay := DontCare
253    isFastPath := DontCare
254    isFastReplay := DontCare
255    handledByMSHR := DontCare
256    replacementUpdated := DontCare
257    missDbUpdated := DontCare
258    delayedLoadError := DontCare
259    lateKill := DontCare
260    feedbacked := DontCare
261    ldCancel := DontCare
262  }
263
264  def asPrefetchReqBundle(): PrefetchReqBundle = {
265    val res = Wire(new PrefetchReqBundle)
266    res.vaddr       := this.vaddr
267    res.paddr       := this.paddr
268    res.pc          := this.uop.pc
269    res.miss        := this.miss
270    res.pfHitStream := isFromStream(this.meta_prefetch)
271
272    res
273  }
274}
275
276class StPrefetchTrainBundle(implicit p: Parameters) extends LdPrefetchTrainBundle {}
277
278class LqWriteBundle(implicit p: Parameters) extends LsPipelineBundle {
279  // load inst replay informations
280  val rep_info = new LoadToLsqReplayIO
281  // queue entry data, except flag bits, will be updated if writeQueue is true,
282  // valid bit in LqWriteBundle will be ignored
283  val data_wen_dup = Vec(6, Bool()) // dirty reg dup
284
285
286  def fromLsPipelineBundle(input: LsPipelineBundle, latch: Boolean = false, enable: Bool = true.B) = {
287    if(latch) vaddr := RegEnable(input.vaddr, enable) else vaddr := input.vaddr
288    if(latch) fullva := RegEnable(input.fullva, enable) else fullva := input.fullva
289    if(latch) vaNeedExt := RegEnable(input.vaNeedExt, enable) else vaNeedExt := input.vaNeedExt
290    if(latch) isHyper := RegEnable(input.isHyper, enable) else isHyper := input.isHyper
291    if(latch) paddr := RegEnable(input.paddr, enable) else paddr := input.paddr
292    if(latch) gpaddr := RegEnable(input.gpaddr, enable) else gpaddr := input.gpaddr
293    if(latch) isForVSnonLeafPTE := RegEnable(input.isForVSnonLeafPTE, enable) else isForVSnonLeafPTE := input.isForVSnonLeafPTE
294    if(latch) mask := RegEnable(input.mask, enable) else mask := input.mask
295    if(latch) data := RegEnable(input.data, enable) else data := input.data
296    if(latch) uop := RegEnable(input.uop, enable) else uop := input.uop
297    if(latch) wlineflag := RegEnable(input.wlineflag, enable) else wlineflag := input.wlineflag
298    if(latch) miss := RegEnable(input.miss, enable) else miss := input.miss
299    if(latch) tlbMiss := RegEnable(input.tlbMiss, enable) else tlbMiss := input.tlbMiss
300    if(latch) ptwBack := RegEnable(input.ptwBack, enable) else ptwBack := input.ptwBack
301    if(latch) mmio := RegEnable(input.mmio, enable) else mmio := input.mmio
302    if(latch) atomic := RegEnable(input.atomic, enable) else atomic := input.atomic
303    if(latch) forwardMask := RegEnable(input.forwardMask, enable) else forwardMask := input.forwardMask
304    if(latch) forwardData := RegEnable(input.forwardData, enable) else forwardData := input.forwardData
305    if(latch) isPrefetch := RegEnable(input.isPrefetch, enable) else isPrefetch := input.isPrefetch
306    if(latch) isHWPrefetch := RegEnable(input.isHWPrefetch, enable) else isHWPrefetch := input.isHWPrefetch
307    if(latch) isFrmMisAlignBuf := RegEnable(input.isFrmMisAlignBuf, enable) else isFrmMisAlignBuf := input.isFrmMisAlignBuf
308    if(latch) isFirstIssue := RegEnable(input.isFirstIssue, enable) else isFirstIssue := input.isFirstIssue
309    if(latch) hasROBEntry := RegEnable(input.hasROBEntry, enable) else hasROBEntry := input.hasROBEntry
310    if(latch) isLoadReplay := RegEnable(input.isLoadReplay, enable) else isLoadReplay := input.isLoadReplay
311    if(latch) isFastPath := RegEnable(input.isFastPath, enable) else isFastPath := input.isFastPath
312    if(latch) isFastReplay := RegEnable(input.isFastReplay, enable) else isFastReplay := input.isFastReplay
313    if(latch) mshrid := RegEnable(input.mshrid, enable) else mshrid := input.mshrid
314    if(latch) forward_tlDchannel := RegEnable(input.forward_tlDchannel, enable) else forward_tlDchannel := input.forward_tlDchannel
315    if(latch) replayCarry := RegEnable(input.replayCarry, enable) else replayCarry := input.replayCarry
316    if(latch) dcacheRequireReplay := RegEnable(input.dcacheRequireReplay, enable) else dcacheRequireReplay := input.dcacheRequireReplay
317    if(latch) schedIndex := RegEnable(input.schedIndex, enable) else schedIndex := input.schedIndex
318    if(latch) handledByMSHR := RegEnable(input.handledByMSHR, enable) else handledByMSHR := input.handledByMSHR
319    if(latch) replacementUpdated := RegEnable(input.replacementUpdated, enable) else replacementUpdated := input.replacementUpdated
320    if(latch) missDbUpdated := RegEnable(input.missDbUpdated, enable) else missDbUpdated := input.missDbUpdated
321    if(latch) delayedLoadError := RegEnable(input.delayedLoadError, enable) else delayedLoadError := input.delayedLoadError
322    if(latch) lateKill := RegEnable(input.lateKill, enable) else lateKill := input.lateKill
323    if(latch) feedbacked := RegEnable(input.feedbacked, enable) else feedbacked := input.feedbacked
324    if(latch) isvec               := RegEnable(input.isvec, enable)               else isvec               := input.isvec
325    if(latch) is128bit            := RegEnable(input.is128bit, enable)            else is128bit            := input.is128bit
326    if(latch) vecActive           := RegEnable(input.vecActive, enable)           else vecActive           := input.vecActive
327    if(latch) uop_unit_stride_fof := RegEnable(input.uop_unit_stride_fof, enable) else uop_unit_stride_fof := input.uop_unit_stride_fof
328    if(latch) reg_offset          := RegEnable(input.reg_offset, enable)          else reg_offset          := input.reg_offset
329    if(latch) mbIndex             := RegEnable(input.mbIndex, enable)             else mbIndex             := input.mbIndex
330    if(latch) elemIdxInsideVd     := RegEnable(input.elemIdxInsideVd, enable)     else elemIdxInsideVd     := input.elemIdxInsideVd
331
332    rep_info := DontCare
333    data_wen_dup := DontCare
334  }
335}
336
337class SqWriteBundle(implicit p: Parameters) extends LsPipelineBundle {
338  val need_rep = Bool()
339}
340
341class LoadForwardQueryIO(implicit p: Parameters) extends XSBundle {
342  val vaddr = Output(UInt(VAddrBits.W))
343  val paddr = Output(UInt(PAddrBits.W))
344  val mask = Output(UInt((VLEN/8).W))
345  val uop = Output(new DynInst) // for replay
346  val pc = Output(UInt(VAddrBits.W)) //for debug
347  val valid = Output(Bool())
348
349  val forwardMaskFast = Input(Vec((VLEN/8), Bool())) // resp to load_s1
350  val forwardMask = Input(Vec((VLEN/8), Bool())) // resp to load_s2
351  val forwardData = Input(Vec((VLEN/8), UInt(8.W))) // resp to load_s2
352
353  // val lqIdx = Output(UInt(LoadQueueIdxWidth.W))
354  val sqIdx = Output(new SqPtr)
355
356  // dataInvalid suggests store to load forward found forward should happen,
357  // but data is not available for now. If dataInvalid, load inst should
358  // be replayed from RS. Feedback type should be RSFeedbackType.dataInvalid
359  val dataInvalid = Input(Bool()) // Addr match, but data is not valid for now
360
361  // matchInvalid suggests in store to load forward logic, paddr cam result does
362  // to equal to vaddr cam result. If matchInvalid, a microarchitectural exception
363  // should be raised to flush SQ and committed sbuffer.
364  val matchInvalid = Input(Bool()) // resp to load_s2
365
366  // addrInvalid suggests store to load forward found forward should happen,
367  // but address (SSID) is not available for now. If addrInvalid, load inst should
368  // be replayed from RS. Feedback type should be RSFeedbackType.addrInvalid
369  val addrInvalid = Input(Bool())
370}
371
372// LoadForwardQueryIO used in load pipeline
373//
374// Difference between PipeLoadForwardQueryIO and LoadForwardQueryIO:
375// PipeIO use predecoded sqIdxMask for better forward timing
376class PipeLoadForwardQueryIO(implicit p: Parameters) extends LoadForwardQueryIO {
377  // val sqIdx = Output(new SqPtr) // for debug, should not be used in pipeline for timing reasons
378  // sqIdxMask is calcuated in earlier stage for better timing
379  val sqIdxMask = Output(UInt(StoreQueueSize.W))
380
381  // dataInvalid: addr match, but data is not valid for now
382  val dataInvalidFast = Input(Bool()) // resp to load_s1
383  // val dataInvalid = Input(Bool()) // resp to load_s2
384  val dataInvalidSqIdx = Input(new SqPtr) // resp to load_s2, sqIdx
385  val addrInvalidSqIdx = Input(new SqPtr) // resp to load_s2, sqIdx
386}
387
388// Query load queue for ld-ld violation
389//
390// Req should be send in load_s1
391// Resp will be generated 1 cycle later
392//
393// Note that query req may be !ready, as dcache is releasing a block
394// If it happens, a replay from rs is needed.
395class LoadNukeQueryReq(implicit p: Parameters) extends XSBundle { // provide lqIdx
396  val uop = new DynInst
397  // mask: load's data mask.
398  val mask = UInt((VLEN/8).W)
399
400  // paddr: load's paddr.
401  val paddr      = UInt(PAddrBits.W)
402  // dataInvalid: load data is invalid.
403  val data_valid = Bool()
404  // nc: is NC access
405  val is_nc = Bool()
406}
407
408class LoadNukeQueryResp(implicit p: Parameters) extends XSBundle {
409  // rep_frm_fetch: ld-ld violation check success, replay from fetch.
410  val rep_frm_fetch = Bool()
411}
412
413class LoadNukeQueryIO(implicit p: Parameters) extends XSBundle {
414  val req    = Decoupled(new LoadNukeQueryReq)
415  val resp   = Flipped(Valid(new LoadNukeQueryResp))
416  val revoke = Output(Bool())
417}
418
419class StoreNukeQueryIO(implicit p: Parameters) extends XSBundle {
420  //  robIdx: Requestor's (a store instruction) rob index for match logic.
421  val robIdx = new RobPtr
422
423  //  paddr: requestor's (a store instruction) physical address for match logic.
424  val paddr  = UInt(PAddrBits.W)
425
426  //  mask: requestor's (a store instruction) data width mask for match logic.
427  val mask = UInt((VLEN/8).W)
428
429  // matchLine: if store is vector 128-bits, load unit need to compare 128-bits vaddr.
430  val matchLine = Bool()
431}
432
433class StoreMaBufToSqControlIO(implicit p: Parameters) extends XSBundle {
434  // from storeMisalignBuffer to storeQueue, control it's sbuffer write
435  val toStoreQueue = Output(new XSBundle {
436    // This entry is a cross page
437    val crossPageWithHit = Bool()
438    val crossPageCanDeq  = Bool()
439    // High page Paddr
440    val paddr = UInt(PAddrBits.W)
441
442    val withSameUop = Bool()
443  })
444  // from storeQueue to storeMisalignBuffer, provide detail info of this store
445  val toStoreMisalignBuffer = Input(new XSBundle {
446    val sqPtr = new SqPtr
447    val doDeq = Bool()
448
449    val uop = new DynInst()
450  })
451}
452
453class StoreMaBufToVecStoreMergeBufferIO(implicit p: Parameters)  extends VLSUBundle{
454  val mbIndex = Output(UInt(vsmBindexBits.W))
455  val flush   = Output(Bool())
456}
457
458// Store byte valid mask write bundle
459//
460// Store byte valid mask write to SQ takes 2 cycles
461class StoreMaskBundle(implicit p: Parameters) extends XSBundle {
462  val sqIdx = new SqPtr
463  val mask = UInt((VLEN/8).W)
464}
465
466class LoadDataFromDcacheBundle(implicit p: Parameters) extends DCacheBundle {
467  // old dcache: optimize data sram read fanout
468  // val bankedDcacheData = Vec(DCacheBanks, UInt(64.W))
469  // val bank_oh = UInt(DCacheBanks.W)
470
471  // new dcache
472  val respDcacheData = UInt(VLEN.W)
473  val forwardMask = Vec(VLEN/8, Bool())
474  val forwardData = Vec(VLEN/8, UInt(8.W))
475  val uop = new DynInst // for data selection, only fwen and fuOpType are used
476  val addrOffset = UInt(4.W) // for data selection
477
478  // forward tilelink D channel
479  val forward_D = Bool()
480  val forwardData_D = Vec(VLEN/8, UInt(8.W))
481
482  // forward mshr data
483  val forward_mshr = Bool()
484  val forwardData_mshr = Vec(VLEN/8, UInt(8.W))
485
486  val forward_result_valid = Bool()
487
488  def mergeTLData(): UInt = {
489    // merge TL D or MSHR data at load s2
490    val dcache_data = respDcacheData
491    val use_D = forward_D && forward_result_valid
492    val use_mshr = forward_mshr && forward_result_valid
493    Mux(
494      use_D || use_mshr,
495      Mux(
496        use_D,
497        forwardData_D.asUInt,
498        forwardData_mshr.asUInt
499      ),
500      dcache_data
501    )
502  }
503
504  def mergeLsqFwdData(dcacheData: UInt): UInt = {
505    // merge dcache and lsq forward data at load s3
506    val rdataVec = VecInit((0 until VLEN / 8).map(j =>
507      Mux(forwardMask(j), forwardData(j), dcacheData(8*(j+1)-1, 8*j))
508    ))
509    rdataVec.asUInt
510  }
511}
512
513// Load writeback data from load queue (refill)
514class LoadDataFromLQBundle(implicit p: Parameters) extends XSBundle {
515  val lqData = UInt(64.W) // load queue has merged data
516  val uop = new DynInst // for data selection, only fwen and fuOpType are used
517  val addrOffset = UInt(3.W) // for data selection
518
519  def mergedData(): UInt = {
520    lqData
521  }
522}
523
524// Bundle for load / store wait waking up
525class MemWaitUpdateReq(implicit p: Parameters) extends XSBundle {
526  val robIdx = Vec(backendParams.StaExuCnt, ValidIO(new RobPtr))
527  val sqIdx = Vec(backendParams.StdCnt, ValidIO(new SqPtr))
528}
529
530object AddPipelineReg {
531  class PipelineRegModule[T <: Data](gen: T) extends Module {
532    val io = IO(new Bundle() {
533      val in = Flipped(DecoupledIO(gen.cloneType))
534      val out = DecoupledIO(gen.cloneType)
535      val isFlush = Input(Bool())
536    })
537
538    val valid = RegInit(false.B)
539    valid.suggestName("pipeline_reg_valid")
540    when (io.out.fire) { valid := false.B }
541    when (io.in.fire) { valid := true.B }
542    when (io.isFlush) { valid := false.B }
543
544    io.in.ready := !valid || io.out.ready
545    io.out.bits := RegEnable(io.in.bits, io.in.fire)
546    io.out.valid := valid //&& !isFlush
547  }
548
549  def apply[T <: Data]
550  (left: DecoupledIO[T], right: DecoupledIO[T], isFlush: Bool,
551   moduleName: Option[String] = None
552  ): Unit = {
553    val pipelineReg = Module(new PipelineRegModule[T](left.bits.cloneType))
554    if(moduleName.nonEmpty) pipelineReg.suggestName(moduleName.get)
555    pipelineReg.io.in <> left
556    right <> pipelineReg.io.out
557    pipelineReg.io.isFlush := isFlush
558  }
559}