1/*************************************************************************************** 2* Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences 3* Copyright (c) 2020-2021 Peng Cheng Laboratory 4* 5* XiangShan is licensed under Mulan PSL v2. 6* You can use this software according to the terms and conditions of the Mulan PSL v2. 7* You may obtain a copy of Mulan PSL v2 at: 8* http://license.coscl.org.cn/MulanPSL2 9* 10* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, 11* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, 12* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. 13* 14* See the Mulan PSL v2 for more details. 15***************************************************************************************/ 16 17package xiangshan.mem 18 19import chipsalliance.rocketchip.config.Parameters 20import chisel3._ 21import chisel3.util._ 22import utils._ 23import utility._ 24import xiangshan._ 25import xiangshan.cache._ 26import xiangshan.cache.{DCacheWordIO, DCacheLineIO, MemoryOpConstants} 27import xiangshan.cache.mmu.{TlbRequestIO} 28import xiangshan.mem._ 29import xiangshan.backend.rob.RobLsqIO 30 31class ExceptionAddrIO(implicit p: Parameters) extends XSBundle { 32 val isStore = Input(Bool()) 33 val vaddr = Output(UInt(VAddrBits.W)) 34} 35 36class FwdEntry extends Bundle { 37 val validFast = Bool() // validFast is generated the same cycle with query 38 val valid = Bool() // valid is generated 1 cycle after query request 39 val data = UInt(8.W) // data is generated 1 cycle after query request 40} 41 42// inflight miss block reqs 43class InflightBlockInfo(implicit p: Parameters) extends XSBundle { 44 val block_addr = UInt(PAddrBits.W) 45 val valid = Bool() 46} 47 48class LsqEnqIO(implicit p: Parameters) extends XSBundle { 49 val canAccept = Output(Bool()) 50 val needAlloc = Vec(exuParameters.LsExuCnt, Input(UInt(2.W))) 51 val req = Vec(exuParameters.LsExuCnt, Flipped(ValidIO(new MicroOp))) 52 val resp = Vec(exuParameters.LsExuCnt, Output(new LSIdx)) 53} 54 55// Load / Store Queue Wrapper for XiangShan Out of Order LSU 56class LsqWrappper(implicit p: Parameters) extends XSModule with HasDCacheParameters with HasPerfEvents { 57 val io = IO(new Bundle() { 58 val hartId = Input(UInt(8.W)) 59 val enq = new LsqEnqIO 60 val brqRedirect = Flipped(ValidIO(new Redirect)) 61 val loadPaddrIn = Vec(LoadPipelineWidth, Flipped(Valid(new LqPaddrWriteBundle))) 62 val loadVaddrIn = Vec(LoadPipelineWidth, Flipped(Valid(new LqVaddrWriteBundle))) 63 val replayFast = Vec(LoadPipelineWidth, Flipped(new LoadToLsqFastIO)) 64 val replaySlow = Vec(LoadPipelineWidth, Flipped(new LoadToLsqSlowIO)) 65 val loadOut = Vec(LoadPipelineWidth, Decoupled(new LsPipelineBundle)) 66 val loadIn = Vec(LoadPipelineWidth, Flipped(Valid(new LqWriteBundle))) 67 val storeIn = Vec(StorePipelineWidth, Flipped(Valid(new LsPipelineBundle))) 68 val storeInRe = Vec(StorePipelineWidth, Input(new LsPipelineBundle())) 69 val storeDataIn = Vec(StorePipelineWidth, Flipped(Valid(new ExuOutput))) // store data, send to sq from rs 70 val storeMaskIn = Vec(StorePipelineWidth, Flipped(Valid(new StoreMaskBundle))) // store mask, send to sq from rs 71 val s2_load_data_forwarded = Vec(LoadPipelineWidth, Input(Bool())) 72 val s3_delayed_load_error = Vec(LoadPipelineWidth, Input(Bool())) 73 val s2_dcache_require_replay = Vec(LoadPipelineWidth, Input(Bool())) 74 val s3_replay_from_fetch = Vec(LoadPipelineWidth, Input(Bool())) 75 val sbuffer = Vec(EnsbufferWidth, Decoupled(new DCacheWordReqWithVaddr)) 76 val ldout = Vec(LoadPipelineWidth, DecoupledIO(new ExuOutput)) // writeback int load 77 val ldRawDataOut = Vec(LoadPipelineWidth, Output(new LoadDataFromLQBundle)) 78 val uncacheOutstanding = Input(Bool()) 79 val mmioStout = DecoupledIO(new ExuOutput) // writeback uncached store 80 val forward = Vec(LoadPipelineWidth, Flipped(new PipeLoadForwardQueryIO)) 81 val loadViolationQuery = Vec(LoadPipelineWidth, Flipped(new LoadViolationQueryIO)) 82 val rob = Flipped(new RobLsqIO) 83 val rollback = Output(Valid(new Redirect)) 84 val refill = Flipped(ValidIO(new Refill)) 85 val release = Flipped(ValidIO(new Release)) 86 val uncache = new UncacheWordIO 87 val exceptionAddr = new ExceptionAddrIO 88 val sqempty = Output(Bool()) 89 val issuePtrExt = Output(new SqPtr) 90 val sqFull = Output(Bool()) 91 val lqFull = Output(Bool()) 92 val lqCancelCnt = Output(UInt(log2Up(LoadQueueSize + 1).W)) 93 val sqCancelCnt = Output(UInt(log2Up(StoreQueueSize + 1).W)) 94 val sqDeq = Output(UInt(log2Ceil(EnsbufferWidth + 1).W)) 95 val trigger = Vec(LoadPipelineWidth, new LqTriggerIO) 96 }) 97 98 val loadQueue = Module(new LoadQueue) 99 val storeQueue = Module(new StoreQueue) 100 101 storeQueue.io.hartId := io.hartId 102 storeQueue.io.uncacheOutstanding := io.uncacheOutstanding 103 104 loadQueue.io.storeDataValidVec := storeQueue.io.storeDataValidVec 105 106 dontTouch(loadQueue.io.tlbReplayDelayCycleCtrl) 107 val tlbReplayDelayCycleCtrl = WireInit(VecInit(Seq(11.U(ReSelectLen.W), 50.U(ReSelectLen.W), 30.U(ReSelectLen.W), 10.U(ReSelectLen.W)))) 108 loadQueue.io.tlbReplayDelayCycleCtrl := tlbReplayDelayCycleCtrl 109 110 // io.enq logic 111 // LSQ: send out canAccept when both load queue and store queue are ready 112 // Dispatch: send instructions to LSQ only when they are ready 113 io.enq.canAccept := loadQueue.io.enq.canAccept && storeQueue.io.enq.canAccept 114 loadQueue.io.enq.sqCanAccept := storeQueue.io.enq.canAccept 115 storeQueue.io.enq.lqCanAccept := loadQueue.io.enq.canAccept 116 for (i <- io.enq.req.indices) { 117 loadQueue.io.enq.needAlloc(i) := io.enq.needAlloc(i)(0) 118 loadQueue.io.enq.req(i).valid := io.enq.needAlloc(i)(0) && io.enq.req(i).valid 119 loadQueue.io.enq.req(i).bits := io.enq.req(i).bits 120 loadQueue.io.enq.req(i).bits.sqIdx := storeQueue.io.enq.resp(i) 121 122 storeQueue.io.enq.needAlloc(i) := io.enq.needAlloc(i)(1) 123 storeQueue.io.enq.req(i).valid := io.enq.needAlloc(i)(1) && io.enq.req(i).valid 124 storeQueue.io.enq.req(i).bits := io.enq.req(i).bits 125 storeQueue.io.enq.req(i).bits := io.enq.req(i).bits 126 storeQueue.io.enq.req(i).bits.lqIdx := loadQueue.io.enq.resp(i) 127 128 io.enq.resp(i).lqIdx := loadQueue.io.enq.resp(i) 129 io.enq.resp(i).sqIdx := storeQueue.io.enq.resp(i) 130 } 131 132 // load queue wiring 133 loadQueue.io.brqRedirect <> io.brqRedirect 134 loadQueue.io.loadPaddrIn <> io.loadPaddrIn 135 loadQueue.io.loadOut <> io.loadOut 136 loadQueue.io.loadVaddrIn <> io.loadVaddrIn 137 loadQueue.io.replayFast <> io.replayFast 138 loadQueue.io.replaySlow <> io.replaySlow 139 loadQueue.io.loadIn <> io.loadIn 140 loadQueue.io.storeIn <> io.storeIn 141 loadQueue.io.s2_load_data_forwarded <> io.s2_load_data_forwarded 142 loadQueue.io.s3_delayed_load_error <> io.s3_delayed_load_error 143 loadQueue.io.s2_dcache_require_replay <> io.s2_dcache_require_replay 144 loadQueue.io.s3_replay_from_fetch <> io.s3_replay_from_fetch 145 loadQueue.io.ldout <> io.ldout 146 loadQueue.io.ldRawDataOut <> io.ldRawDataOut 147 loadQueue.io.rob <> io.rob 148 loadQueue.io.rollback <> io.rollback 149 loadQueue.io.refill <> io.refill 150 loadQueue.io.release <> io.release 151 loadQueue.io.trigger <> io.trigger 152 loadQueue.io.exceptionAddr.isStore := DontCare 153 loadQueue.io.lqCancelCnt <> io.lqCancelCnt 154 155 // store queue wiring 156 // storeQueue.io <> DontCare 157 storeQueue.io.brqRedirect <> io.brqRedirect 158 storeQueue.io.storeIn <> io.storeIn 159 storeQueue.io.storeInRe <> io.storeInRe 160 storeQueue.io.storeDataIn <> io.storeDataIn 161 storeQueue.io.storeMaskIn <> io.storeMaskIn 162 storeQueue.io.sbuffer <> io.sbuffer 163 storeQueue.io.mmioStout <> io.mmioStout 164 storeQueue.io.rob <> io.rob 165 storeQueue.io.exceptionAddr.isStore := DontCare 166 storeQueue.io.issuePtrExt <> io.issuePtrExt 167 storeQueue.io.sqCancelCnt <> io.sqCancelCnt 168 storeQueue.io.sqDeq <> io.sqDeq 169 170 loadQueue.io.load_s1 <> io.forward 171 storeQueue.io.forward <> io.forward // overlap forwardMask & forwardData, DO NOT CHANGE SEQUENCE 172 173 loadQueue.io.loadViolationQuery <> io.loadViolationQuery 174 175 storeQueue.io.sqempty <> io.sqempty 176 177 // rob commits for lsq is delayed for two cycles, which causes the delayed update for deqPtr in lq/sq 178 // s0: commit 179 // s1: exception find 180 // s2: exception triggered 181 // s3: ptr updated & new address 182 // address will be used at the next cycle after exception is triggered 183 io.exceptionAddr.vaddr := Mux(RegNext(io.exceptionAddr.isStore), storeQueue.io.exceptionAddr.vaddr, loadQueue.io.exceptionAddr.vaddr) 184 185 // naive uncache arbiter 186 val s_idle :: s_load :: s_store :: Nil = Enum(3) 187 val pendingstate = RegInit(s_idle) 188 189 switch(pendingstate){ 190 is(s_idle){ 191 when(io.uncache.req.fire() && !io.uncacheOutstanding){ 192 pendingstate := Mux(loadQueue.io.uncache.req.valid, s_load, 193 Mux(io.uncacheOutstanding, s_idle, s_store)) 194 } 195 } 196 is(s_load){ 197 when(io.uncache.resp.fire()){ 198 pendingstate := s_idle 199 } 200 } 201 is(s_store){ 202 when(io.uncache.resp.fire()){ 203 pendingstate := s_idle 204 } 205 } 206 } 207 208 loadQueue.io.uncache := DontCare 209 storeQueue.io.uncache := DontCare 210 loadQueue.io.uncache.resp.valid := false.B 211 storeQueue.io.uncache.resp.valid := false.B 212 when(loadQueue.io.uncache.req.valid){ 213 io.uncache.req <> loadQueue.io.uncache.req 214 }.otherwise{ 215 io.uncache.req <> storeQueue.io.uncache.req 216 } 217 when (io.uncacheOutstanding) { 218 io.uncache.resp <> loadQueue.io.uncache.resp 219 } .otherwise { 220 when(pendingstate === s_load){ 221 io.uncache.resp <> loadQueue.io.uncache.resp 222 }.otherwise{ 223 io.uncache.resp <> storeQueue.io.uncache.resp 224 } 225 } 226 227 228 assert(!(loadQueue.io.uncache.req.valid && storeQueue.io.uncache.req.valid)) 229 assert(!(loadQueue.io.uncache.resp.valid && storeQueue.io.uncache.resp.valid)) 230 when (!io.uncacheOutstanding) { 231 assert(!((loadQueue.io.uncache.resp.valid || storeQueue.io.uncache.resp.valid) && pendingstate === s_idle)) 232 } 233 234 io.lqFull := loadQueue.io.lqFull 235 io.sqFull := storeQueue.io.sqFull 236 237 val perfEvents = Seq(loadQueue, storeQueue).flatMap(_.getPerfEvents) 238 generatePerfEvent() 239} 240 241class LsqEnqCtrl(implicit p: Parameters) extends XSModule { 242 val io = IO(new Bundle { 243 val redirect = Flipped(ValidIO(new Redirect)) 244 // to dispatch 245 val enq = new LsqEnqIO 246 // from rob 247 val lcommit = Input(UInt(log2Up(CommitWidth + 1).W)) 248 // from `memBlock.io.sqDeq` 249 val scommit = Input(UInt(log2Ceil(EnsbufferWidth + 1).W)) 250 // from/tp lsq 251 val lqCancelCnt = Input(UInt(log2Up(LoadQueueSize + 1).W)) 252 val sqCancelCnt = Input(UInt(log2Up(StoreQueueSize + 1).W)) 253 val enqLsq = Flipped(new LsqEnqIO) 254 }) 255 256 val lqPtr = RegInit(0.U.asTypeOf(new LqPtr)) 257 val sqPtr = RegInit(0.U.asTypeOf(new SqPtr)) 258 val lqCounter = RegInit(LoadQueueSize.U(log2Up(LoadQueueSize + 1).W)) 259 val sqCounter = RegInit(StoreQueueSize.U(log2Up(StoreQueueSize + 1).W)) 260 val canAccept = RegInit(false.B) 261 262 val loadEnqNumber = PopCount(io.enq.req.zip(io.enq.needAlloc).map(x => x._1.valid && x._2(0))) 263 val storeEnqNumber = PopCount(io.enq.req.zip(io.enq.needAlloc).map(x => x._1.valid && x._2(1))) 264 265 // How to update ptr and counter: 266 // (1) by default, updated according to enq/commit 267 // (2) when redirect and dispatch queue is empty, update according to lsq 268 val t1_redirect = RegNext(io.redirect.valid) 269 val t2_redirect = RegNext(t1_redirect) 270 val t2_update = t2_redirect && !VecInit(io.enq.needAlloc.map(_.orR)).asUInt.orR 271 val t3_update = RegNext(t2_update) 272 val t3_lqCancelCnt = RegNext(io.lqCancelCnt) 273 val t3_sqCancelCnt = RegNext(io.sqCancelCnt) 274 when (t3_update) { 275 lqPtr := lqPtr - t3_lqCancelCnt 276 lqCounter := lqCounter + io.lcommit + t3_lqCancelCnt 277 sqPtr := sqPtr - t3_sqCancelCnt 278 sqCounter := sqCounter + io.scommit + t3_sqCancelCnt 279 }.elsewhen (!io.redirect.valid && io.enq.canAccept) { 280 lqPtr := lqPtr + loadEnqNumber 281 lqCounter := lqCounter + io.lcommit - loadEnqNumber 282 sqPtr := sqPtr + storeEnqNumber 283 sqCounter := sqCounter + io.scommit - storeEnqNumber 284 }.otherwise { 285 lqCounter := lqCounter + io.lcommit 286 sqCounter := sqCounter + io.scommit 287 } 288 289 290 val maxAllocate = Seq(exuParameters.LduCnt, exuParameters.StuCnt).max 291 val ldCanAccept = lqCounter >= loadEnqNumber +& maxAllocate.U 292 val sqCanAccept = sqCounter >= storeEnqNumber +& maxAllocate.U 293 // It is possible that t3_update and enq are true at the same clock cycle. 294 // For example, if redirect.valid lasts more than one clock cycle, 295 // after the last redirect, new instructions may enter but previously redirect 296 // has not been resolved (updated according to the cancel count from LSQ). 297 // To solve the issue easily, we block enqueue when t3_update, which is RegNext(t2_update). 298 io.enq.canAccept := RegNext(ldCanAccept && sqCanAccept && !t2_update) 299 val lqOffset = Wire(Vec(io.enq.resp.length, UInt(log2Up(maxAllocate + 1).W))) 300 val sqOffset = Wire(Vec(io.enq.resp.length, UInt(log2Up(maxAllocate + 1).W))) 301 for ((resp, i) <- io.enq.resp.zipWithIndex) { 302 lqOffset(i) := PopCount(io.enq.needAlloc.take(i).map(a => a(0))) 303 resp.lqIdx := lqPtr + lqOffset(i) 304 sqOffset(i) := PopCount(io.enq.needAlloc.take(i).map(a => a(1))) 305 resp.sqIdx := sqPtr + sqOffset(i) 306 } 307 308 io.enqLsq.needAlloc := RegNext(io.enq.needAlloc) 309 io.enqLsq.req.zip(io.enq.req).zip(io.enq.resp).foreach{ case ((toLsq, enq), resp) => 310 val do_enq = enq.valid && !io.redirect.valid && io.enq.canAccept 311 toLsq.valid := RegNext(do_enq) 312 toLsq.bits := RegEnable(enq.bits, do_enq) 313 toLsq.bits.lqIdx := RegEnable(resp.lqIdx, do_enq) 314 toLsq.bits.sqIdx := RegEnable(resp.sqIdx, do_enq) 315 } 316 317} 318