1/*************************************************************************************** 2* Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences 3* Copyright (c) 2020-2021 Peng Cheng Laboratory 4* 5* XiangShan is licensed under Mulan PSL v2. 6* You can use this software according to the terms and conditions of the Mulan PSL v2. 7* You may obtain a copy of Mulan PSL v2 at: 8* http://license.coscl.org.cn/MulanPSL2 9* 10* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, 11* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, 12* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. 13* 14* See the Mulan PSL v2 for more details. 15***************************************************************************************/ 16 17package xiangshan.mem 18 19import org.chipsalliance.cde.config.Parameters 20import chisel3._ 21import chisel3.util._ 22import utils._ 23import utility._ 24import xiangshan.ExceptionNO._ 25import xiangshan._ 26import xiangshan.backend.Bundles.{MemExuInput, MemExuOutput} 27import xiangshan.backend.fu.PMPRespBundle 28import xiangshan.backend.fu.FuConfig._ 29import xiangshan.backend.fu.FuType._ 30import xiangshan.backend.ctrlblock.DebugLsInfoBundle 31import xiangshan.backend.fu.NewCSR._ 32import xiangshan.cache.mmu.{TlbCmd, TlbReq, TlbRequestIO, TlbResp, Pbmt} 33import xiangshan.cache.{DcacheStoreRequestIO, DCacheStoreIO, MemoryOpConstants, HasDCacheParameters, StorePrefetchReq} 34 35class StoreUnit(implicit p: Parameters) extends XSModule 36 with HasDCacheParameters 37 with HasVLSUParameters 38 { 39 val io = IO(new Bundle() { 40 val redirect = Flipped(ValidIO(new Redirect)) 41 val csrCtrl = Flipped(new CustomCSRCtrlIO) 42 val stin = Flipped(Decoupled(new MemExuInput)) 43 val issue = Valid(new MemExuInput) 44 // misalignBuffer issue path 45 val misalign_stin = Flipped(Decoupled(new LsPipelineBundle)) 46 val misalign_stout = Valid(new SqWriteBundle) 47 val tlb = new TlbRequestIO() 48 val dcache = new DCacheStoreIO 49 val pmp = Flipped(new PMPRespBundle()) 50 val lsq = ValidIO(new LsPipelineBundle) 51 val lsq_replenish = Output(new LsPipelineBundle()) 52 val feedback_slow = ValidIO(new RSFeedback) 53 val prefetch_req = Flipped(DecoupledIO(new StorePrefetchReq)) 54 // provide prefetch info to sms 55 val prefetch_train = ValidIO(new StPrefetchTrainBundle()) 56 // speculative for gated control 57 val s1_prefetch_spec = Output(Bool()) 58 val s2_prefetch_spec = Output(Bool()) 59 val stld_nuke_query = Valid(new StoreNukeQueryIO) 60 val stout = DecoupledIO(new MemExuOutput) // writeback store 61 val vecstout = DecoupledIO(new VecPipelineFeedbackIO(isVStore = true)) 62 // store mask, send to sq in store_s0 63 val st_mask_out = Valid(new StoreMaskBundle) 64 val debug_ls = Output(new DebugLsInfoBundle) 65 // vector 66 val vecstin = Flipped(Decoupled(new VecPipeBundle(isVStore = true))) 67 val vec_isFirstIssue = Input(Bool()) 68 // writeback to misalign buffer 69 val misalign_buf = Valid(new LsPipelineBundle) 70 // trigger 71 val fromCsrTrigger = Input(new CsrTriggerBundle) 72 }) 73 74 val s1_ready, s2_ready, s3_ready = WireInit(false.B) 75 76 // Pipeline 77 // -------------------------------------------------------------------------------- 78 // stage 0 79 // -------------------------------------------------------------------------------- 80 // generate addr, use addr to query DCache and DTLB 81 val s0_iss_valid = io.stin.valid 82 val s0_prf_valid = io.prefetch_req.valid && io.dcache.req.ready 83 val s0_vec_valid = io.vecstin.valid 84 val s0_ma_st_valid = io.misalign_stin.valid 85 val s0_valid = s0_iss_valid || s0_prf_valid || s0_vec_valid || s0_ma_st_valid 86 val s0_use_flow_ma = s0_ma_st_valid 87 val s0_use_flow_vec = s0_vec_valid && !s0_ma_st_valid 88 val s0_use_flow_rs = s0_iss_valid && !s0_vec_valid && !s0_ma_st_valid 89 val s0_use_flow_prf = s0_prf_valid && !s0_iss_valid && !s0_vec_valid && !s0_ma_st_valid 90 val s0_use_non_prf_flow = s0_use_flow_rs || s0_use_flow_vec || s0_use_flow_ma 91 val s0_stin = Mux(s0_use_flow_rs, io.stin.bits, 0.U.asTypeOf(io.stin.bits)) 92 val s0_vecstin = Mux(s0_use_flow_vec, io.vecstin.bits, 0.U.asTypeOf(io.vecstin.bits)) 93 val s0_uop = Mux( 94 s0_use_flow_ma, 95 io.misalign_stin.bits.uop, 96 Mux( 97 s0_use_flow_rs, 98 s0_stin.uop, 99 s0_vecstin.uop 100 ) 101 ) 102 val s0_isFirstIssue = Mux( 103 s0_use_flow_ma, 104 false.B, 105 s0_use_flow_rs && io.stin.bits.isFirstIssue || s0_use_flow_vec && io.vec_isFirstIssue 106 ) 107 val s0_size = Mux(s0_use_non_prf_flow, s0_uop.fuOpType(2,0), 0.U)// may broken if use it in feature 108 val s0_mem_idx = Mux(s0_use_non_prf_flow, s0_uop.sqIdx.value, 0.U) 109 val s0_rob_idx = Mux(s0_use_non_prf_flow, s0_uop.robIdx, 0.U.asTypeOf(s0_uop.robIdx)) 110 val s0_pc = Mux(s0_use_non_prf_flow, s0_uop.pc, 0.U) 111 val s0_instr_type = Mux(s0_use_non_prf_flow, STORE_SOURCE.U, DCACHE_PREFETCH_SOURCE.U) 112 val s0_wlineflag = Mux(s0_use_flow_rs, s0_uop.fuOpType === LSUOpType.cbo_zero, false.B) 113 val s0_out = Wire(new LsPipelineBundle) 114 val s0_kill = s0_uop.robIdx.needFlush(io.redirect) 115 val s0_can_go = s1_ready 116 val s0_fire = s0_valid && !s0_kill && s0_can_go 117 val s0_is128bit = Mux(s0_use_flow_ma, io.misalign_stin.bits.is128bit, is128Bit(s0_vecstin.alignedType)) 118 // vector 119 val s0_vecActive = !s0_use_flow_vec || s0_vecstin.vecActive 120 // val s0_flowPtr = s0_vecstin.flowPtr 121 // val s0_isLastElem = s0_vecstin.isLastElem 122 val s0_secondInv = s0_vecstin.usSecondInv 123 val s0_elemIdx = s0_vecstin.elemIdx 124 val s0_alignedType = s0_vecstin.alignedType 125 val s0_mBIndex = s0_vecstin.mBIndex 126 val s0_vecBaseVaddr = s0_vecstin.basevaddr 127 128 // generate addr 129 val s0_saddr = s0_stin.src(0) + SignExt(s0_stin.uop.imm(11,0), VAddrBits) 130 val s0_fullva = Wire(UInt(XLEN.W)) 131 val s0_vaddr = Mux( 132 s0_use_flow_ma, 133 io.misalign_stin.bits.vaddr, 134 Mux( 135 s0_use_flow_rs, 136 s0_saddr, 137 Mux( 138 s0_use_flow_vec, 139 s0_vecstin.vaddr(VAddrBits - 1, 0), 140 io.prefetch_req.bits.vaddr 141 ) 142 ) 143 ) 144 s0_fullva := Mux( 145 s0_use_flow_rs, 146 s0_stin.src(0) + SignExt(s0_stin.uop.imm(11,0), XLEN), 147 Mux( 148 s0_use_flow_vec, 149 s0_vecstin.vaddr, 150 s0_vaddr 151 ) 152 ) 153 154 val s0_mask = Mux( 155 s0_use_flow_ma, 156 io.misalign_stin.bits.mask, 157 Mux( 158 s0_use_flow_rs, 159 genVWmask128(s0_saddr, s0_uop.fuOpType(2,0)), 160 Mux( 161 s0_use_flow_vec, 162 s0_vecstin.mask, 163 // -1.asSInt.asUInt 164 Fill(VLEN/8, 1.U(1.W)) 165 ) 166 ) 167 ) 168 169 io.tlb.req.valid := s0_valid 170 io.tlb.req.bits.vaddr := s0_vaddr 171 io.tlb.req.bits.fullva := s0_fullva 172 io.tlb.req.bits.checkfullva := s0_use_flow_rs || s0_use_flow_vec 173 io.tlb.req.bits.cmd := TlbCmd.write 174 io.tlb.req.bits.isPrefetch := s0_use_flow_prf 175 io.tlb.req.bits.size := s0_size 176 io.tlb.req.bits.kill := false.B 177 io.tlb.req.bits.memidx.is_ld := false.B 178 io.tlb.req.bits.memidx.is_st := true.B 179 io.tlb.req.bits.memidx.idx := s0_mem_idx 180 io.tlb.req.bits.debug.robIdx := s0_rob_idx 181 io.tlb.req.bits.no_translate := false.B 182 io.tlb.req.bits.debug.pc := s0_pc 183 io.tlb.req.bits.debug.isFirstIssue := s0_isFirstIssue 184 io.tlb.req_kill := false.B 185 io.tlb.req.bits.hyperinst := LSUOpType.isHsv(s0_uop.fuOpType) 186 io.tlb.req.bits.hlvx := false.B 187 io.tlb.req.bits.pmp_addr := DontCare 188 189 // Dcache access here: not **real** dcache write 190 // just read meta and tag in dcache, to find out the store will hit or miss 191 192 // NOTE: The store request does not wait for the dcache to be ready. 193 // If the dcache is not ready at this time, the dcache is not queried. 194 // But, store prefetch request will always wait for dcache to be ready to make progress. 195 io.dcache.req.valid := s0_fire 196 io.dcache.req.bits.cmd := MemoryOpConstants.M_PFW 197 io.dcache.req.bits.vaddr := s0_vaddr 198 io.dcache.req.bits.instrtype := s0_instr_type 199 200 s0_out := DontCare 201 s0_out.vaddr := s0_vaddr 202 s0_out.fullva := s0_fullva 203 // Now data use its own io 204 // s1_out.data := genWdata(s1_in.src(1), s1_in.uop.fuOpType(1,0)) 205 s0_out.data := s0_stin.src(1) 206 s0_out.uop := s0_uop 207 s0_out.miss := false.B 208 s0_out.mask := s0_mask 209 s0_out.isFirstIssue := s0_isFirstIssue 210 s0_out.isHWPrefetch := s0_use_flow_prf 211 s0_out.wlineflag := s0_wlineflag 212 s0_out.isvec := s0_use_flow_vec 213 s0_out.is128bit := s0_is128bit 214 s0_out.vecActive := s0_vecActive 215 s0_out.usSecondInv := s0_secondInv 216 s0_out.elemIdx := s0_elemIdx 217 s0_out.alignedType := s0_alignedType 218 s0_out.mbIndex := s0_mBIndex 219 s0_out.vecBaseVaddr := s0_vecBaseVaddr 220 when(s0_valid && s0_isFirstIssue) { 221 s0_out.uop.debugInfo.tlbFirstReqTime := GTimer() 222 } 223 s0_out.isFrmMisAlignBuf := s0_use_flow_ma 224 225 // exception check 226 val s0_addr_aligned = LookupTree(Mux(s0_use_flow_vec, s0_vecstin.alignedType(1,0), s0_uop.fuOpType(1, 0)), List( 227 "b00".U -> true.B, //b 228 "b01".U -> (s0_out.vaddr(0) === 0.U), //h 229 "b10".U -> (s0_out.vaddr(1,0) === 0.U), //w 230 "b11".U -> (s0_out.vaddr(2,0) === 0.U) //d 231 )) 232 // if vector store sends 128-bit requests, its address must be 128-aligned 233 XSError(s0_use_flow_vec && s0_out.vaddr(3, 0) =/= 0.U && s0_vecstin.alignedType(2), "unit stride 128 bit element is not aligned!") 234 s0_out.uop.exceptionVec(storeAddrMisaligned) := Mux(s0_use_non_prf_flow, (!s0_addr_aligned || s0_vecstin.uop.exceptionVec(storeAddrMisaligned) && s0_vecActive), false.B) 235 236 io.st_mask_out.valid := s0_use_flow_rs || s0_use_flow_vec 237 io.st_mask_out.bits.mask := s0_out.mask 238 io.st_mask_out.bits.sqIdx := s0_out.uop.sqIdx 239 240 io.stin.ready := s1_ready && s0_use_flow_rs 241 io.vecstin.ready := s1_ready && s0_use_flow_vec 242 io.prefetch_req.ready := s1_ready && io.dcache.req.ready && !s0_iss_valid && !s0_vec_valid && !s0_ma_st_valid 243 io.misalign_stin.ready := s1_ready && s0_use_flow_ma 244 245 // Pipeline 246 // -------------------------------------------------------------------------------- 247 // stage 1 248 // -------------------------------------------------------------------------------- 249 // TLB resp (send paddr to dcache) 250 val s1_valid = RegInit(false.B) 251 val s1_in = RegEnable(s0_out, s0_fire) 252 val s1_out = Wire(new LsPipelineBundle) 253 val s1_kill = Wire(Bool()) 254 val s1_can_go = s2_ready 255 val s1_fire = s1_valid && !s1_kill && s1_can_go 256 val s1_vecActive = RegEnable(s0_out.vecActive, true.B, s0_fire) 257 val s1_frm_mabuf = s1_in.isFrmMisAlignBuf 258 259 // mmio cbo decoder 260 val s1_mmio_cbo = s1_in.uop.fuOpType === LSUOpType.cbo_clean || 261 s1_in.uop.fuOpType === LSUOpType.cbo_flush || 262 s1_in.uop.fuOpType === LSUOpType.cbo_inval 263 val s1_vaNeedExt = io.tlb.resp.bits.excp(0).vaNeedExt 264 val s1_isHyper = io.tlb.resp.bits.excp(0).isHyper 265 val s1_paddr = io.tlb.resp.bits.paddr(0) 266 val s1_gpaddr = io.tlb.resp.bits.gpaddr(0) 267 val s1_fullva = io.tlb.resp.bits.fullva 268 val s1_isForVSnonLeafPTE = io.tlb.resp.bits.isForVSnonLeafPTE 269 val s1_tlb_miss = io.tlb.resp.bits.miss && io.tlb.resp.valid && s1_valid 270 val s1_mmio = s1_mmio_cbo 271 val s1_pbmt = Mux(!s1_tlb_miss, io.tlb.resp.bits.pbmt.head, 0.U(Pbmt.width.W)) 272 val s1_exception = ExceptionNO.selectByFu(s1_out.uop.exceptionVec, StaCfg).asUInt.orR 273 val s1_isvec = RegEnable(s0_out.isvec, false.B, s0_fire) 274 // val s1_isLastElem = RegEnable(s0_isLastElem, false.B, s0_fire) 275 s1_kill := s1_in.uop.robIdx.needFlush(io.redirect) || (s1_tlb_miss && !s1_isvec && !s1_frm_mabuf) 276 277 s1_ready := !s1_valid || s1_kill || s2_ready 278 io.tlb.resp.ready := true.B // TODO: why dtlbResp needs a ready? 279 when (s0_fire) { s1_valid := true.B } 280 .elsewhen (s1_fire) { s1_valid := false.B } 281 .elsewhen (s1_kill) { s1_valid := false.B } 282 283 // st-ld violation dectect request. 284 io.stld_nuke_query.valid := s1_valid && !s1_tlb_miss && !s1_in.isHWPrefetch && !s1_frm_mabuf 285 io.stld_nuke_query.bits.robIdx := s1_in.uop.robIdx 286 io.stld_nuke_query.bits.paddr := s1_paddr 287 io.stld_nuke_query.bits.mask := s1_in.mask 288 io.stld_nuke_query.bits.matchLine := s1_in.isvec && s1_in.is128bit 289 290 // issue 291 io.issue.valid := s1_valid && !s1_tlb_miss && !s1_in.isHWPrefetch && !s1_isvec && !s1_frm_mabuf 292 io.issue.bits := RegEnable(s0_stin, s0_valid) 293 294 295 // Send TLB feedback to store issue queue 296 // Store feedback is generated in store_s1, sent to RS in store_s2 297 val s1_feedback = Wire(Valid(new RSFeedback)) 298 s1_feedback.valid := s1_valid & !s1_in.isHWPrefetch 299 s1_feedback.bits.hit := !s1_tlb_miss 300 s1_feedback.bits.flushState := io.tlb.resp.bits.ptwBack 301 s1_feedback.bits.robIdx := s1_out.uop.robIdx 302 s1_feedback.bits.sourceType := RSFeedbackType.tlbMiss 303 s1_feedback.bits.dataInvalidSqIdx := DontCare 304 s1_feedback.bits.sqIdx := s1_out.uop.sqIdx 305 s1_feedback.bits.lqIdx := s1_out.uop.lqIdx 306 307 XSDebug(s1_feedback.valid, 308 "S1 Store: tlbHit: %d robIdx: %d\n", 309 s1_feedback.bits.hit, 310 s1_feedback.bits.robIdx.value 311 ) 312 313 // io.feedback_slow := s1_feedback 314 315 // get paddr from dtlb, check if rollback is needed 316 // writeback store inst to lsq 317 s1_out := s1_in 318 s1_out.paddr := s1_paddr 319 s1_out.gpaddr := s1_gpaddr 320 s1_out.fullva := s1_fullva 321 s1_out.vaNeedExt := s1_vaNeedExt 322 s1_out.isHyper := s1_isHyper 323 s1_out.miss := false.B 324 s1_out.nc := Pbmt.isNC(s1_pbmt) 325 s1_out.mmio := s1_mmio || Pbmt.isIO(s1_pbmt) 326 s1_out.tlbMiss := s1_tlb_miss 327 s1_out.atomic := s1_mmio || Pbmt.isIO(s1_pbmt) 328 s1_out.isForVSnonLeafPTE := s1_isForVSnonLeafPTE 329 when (!s1_out.isvec && RegNext(io.tlb.req.bits.checkfullva) && 330 (s1_out.uop.exceptionVec(storePageFault) || 331 s1_out.uop.exceptionVec(storeAccessFault) || 332 s1_out.uop.exceptionVec(storeGuestPageFault))) { 333 s1_out.uop.exceptionVec(storeAddrMisaligned) := false.B 334 } 335 s1_out.uop.exceptionVec(storePageFault) := io.tlb.resp.bits.excp(0).pf.st && s1_vecActive 336 s1_out.uop.exceptionVec(storeAccessFault) := io.tlb.resp.bits.excp(0).af.st && s1_vecActive 337 s1_out.uop.exceptionVec(storeGuestPageFault) := io.tlb.resp.bits.excp(0).gpf.st && s1_vecActive 338 339 // trigger 340 val storeTrigger = Module(new MemTrigger(MemType.STORE)) 341 storeTrigger.io.fromCsrTrigger.tdataVec := io.fromCsrTrigger.tdataVec 342 storeTrigger.io.fromCsrTrigger.tEnableVec := io.fromCsrTrigger.tEnableVec 343 storeTrigger.io.fromCsrTrigger.triggerCanRaiseBpExp := io.fromCsrTrigger.triggerCanRaiseBpExp 344 storeTrigger.io.fromCsrTrigger.debugMode := io.fromCsrTrigger.debugMode 345 storeTrigger.io.fromLoadStore.vaddr := s1_in.vaddr 346 storeTrigger.io.fromLoadStore.isVectorUnitStride := s1_in.isvec && s1_in.is128bit 347 storeTrigger.io.fromLoadStore.mask := s1_in.mask 348 349 val s1_trigger_action = storeTrigger.io.toLoadStore.triggerAction 350 val s1_trigger_debug_mode = TriggerAction.isDmode(s1_trigger_action) 351 val s1_trigger_breakpoint = TriggerAction.isExp(s1_trigger_action) 352 353 s1_out.uop.flushPipe := false.B 354 s1_out.uop.trigger := s1_trigger_action 355 s1_out.uop.exceptionVec(breakPoint) := s1_trigger_breakpoint 356 s1_out.vecVaddrOffset := Mux( 357 s1_trigger_debug_mode || s1_trigger_breakpoint, 358 storeTrigger.io.toLoadStore.triggerVaddr - s1_in.vecBaseVaddr, 359 s1_in.vaddr + genVFirstUnmask(s1_in.mask).asUInt - s1_in.vecBaseVaddr , 360 ) 361 s1_out.vecTriggerMask := Mux(s1_trigger_debug_mode || s1_trigger_breakpoint, storeTrigger.io.toLoadStore.triggerMask, 0.U) 362 363 // scalar store and scalar load nuke check, and also other purposes 364 io.lsq.valid := s1_valid && !s1_in.isHWPrefetch && !s1_frm_mabuf 365 io.lsq.bits := s1_out 366 io.lsq.bits.miss := s1_tlb_miss 367 368 // goto misalignBuffer 369 io.misalign_buf.valid := s1_valid && !s1_tlb_miss && !s1_in.isHWPrefetch && GatedValidRegNext(io.csrCtrl.hd_misalign_st_enable) && !s1_in.isvec 370 io.misalign_buf.bits := io.lsq.bits 371 372 // kill dcache write intent request when tlb miss or exception 373 io.dcache.s1_kill := (s1_tlb_miss || s1_exception || s1_mmio || s1_in.uop.robIdx.needFlush(io.redirect)) 374 io.dcache.s1_paddr := s1_paddr 375 376 // write below io.out.bits assign sentence to prevent overwriting values 377 val s1_tlb_memidx = io.tlb.resp.bits.memidx 378 when(s1_tlb_memidx.is_st && io.tlb.resp.valid && !s1_tlb_miss && s1_tlb_memidx.idx === s1_out.uop.sqIdx.value) { 379 // printf("Store idx = %d\n", s1_tlb_memidx.idx) 380 s1_out.uop.debugInfo.tlbRespTime := GTimer() 381 } 382 383 // Pipeline 384 // -------------------------------------------------------------------------------- 385 // stage 2 386 // -------------------------------------------------------------------------------- 387 // mmio check 388 val s2_valid = RegInit(false.B) 389 val s2_in = RegEnable(s1_out, s1_fire) 390 val s2_out = Wire(new LsPipelineBundle) 391 val s2_kill = Wire(Bool()) 392 val s2_can_go = s3_ready 393 val s2_fire = s2_valid && !s2_kill && s2_can_go 394 val s2_vecActive = RegEnable(s1_out.vecActive, true.B, s1_fire) 395 val s2_frm_mabuf = s2_in.isFrmMisAlignBuf 396 val s2_pbmt = RegEnable(s1_pbmt, s1_fire) 397 val s2_trigger_debug_mode = RegEnable(s1_trigger_debug_mode, false.B, s1_fire) 398 val s2_mis_align = GatedValidRegNext(io.csrCtrl.hd_misalign_st_enable) && !s2_in.isvec && 399 s2_in.uop.exceptionVec(storeAddrMisaligned) && !s2_in.uop.exceptionVec(breakPoint) && !s2_trigger_debug_mode 400 401 s2_ready := !s2_valid || s2_kill || s3_ready 402 when (s1_fire) { s2_valid := true.B } 403 .elsewhen (s2_fire) { s2_valid := false.B } 404 .elsewhen (s2_kill) { s2_valid := false.B } 405 406 val s2_pmp = WireInit(io.pmp) 407 408 val s2_exception = RegNext(s1_feedback.bits.hit) && 409 (s2_trigger_debug_mode || ExceptionNO.selectByFu(s2_out.uop.exceptionVec, StaCfg).asUInt.orR) 410 val s2_mmio = (s2_in.mmio || (Pbmt.isPMA(s2_pbmt) && s2_pmp.mmio)) && RegNext(s1_feedback.bits.hit) 411 val s2_actually_uncache = (Pbmt.isPMA(s2_pbmt) && s2_pmp.mmio || s2_in.nc || s2_in.mmio) && RegNext(s1_feedback.bits.hit) 412 val s2_uncache = !s2_exception && !s2_in.tlbMiss && s2_actually_uncache 413 s2_kill := ((s2_mmio && !s2_exception) && !s2_in.isvec) || s2_in.uop.robIdx.needFlush(io.redirect) 414 415 s2_out := s2_in 416 s2_out.af := s2_out.uop.exceptionVec(storeAccessFault) 417 s2_out.mmio := s2_mmio && !s2_exception 418 s2_out.atomic := s2_in.atomic || Pbmt.isPMA(s2_pbmt) && s2_pmp.atomic 419 s2_out.uop.exceptionVec(storeAccessFault) := (s2_in.uop.exceptionVec(storeAccessFault) || 420 s2_pmp.st || 421 (s2_in.isvec && s2_actually_uncache && RegNext(s1_feedback.bits.hit)) 422 ) && s2_vecActive 423 s2_out.uop.vpu.vstart := s2_in.vecVaddrOffset >> s2_in.uop.vpu.veew 424 425 // kill dcache write intent request when mmio or exception 426 io.dcache.s2_kill := (s2_uncache || s2_exception || s2_in.uop.robIdx.needFlush(io.redirect)) 427 io.dcache.s2_pc := s2_out.uop.pc 428 // TODO: dcache resp 429 io.dcache.resp.ready := true.B 430 431 // feedback tlb miss to RS in store_s2 432 val feedback_slow_valid = WireInit(false.B) 433 feedback_slow_valid := s1_feedback.valid && !s1_out.uop.robIdx.needFlush(io.redirect) && !s1_out.isvec && !s1_frm_mabuf 434 io.feedback_slow.valid := GatedValidRegNext(feedback_slow_valid) 435 io.feedback_slow.bits := RegEnable(s1_feedback.bits, feedback_slow_valid) 436 437 val s2_vecFeedback = RegNext(!s1_out.uop.robIdx.needFlush(io.redirect) && s1_feedback.bits.hit && s1_feedback.valid) && s2_in.isvec 438 439 val s2_misalign_stout = WireInit(0.U.asTypeOf(io.misalign_stout)) 440 s2_misalign_stout.valid := s2_valid && s2_can_go && s2_frm_mabuf 441 s2_misalign_stout.bits.mmio := s2_out.mmio 442 s2_misalign_stout.bits.vaddr := s2_out.vaddr 443 s2_misalign_stout.bits.isHyper := s2_out.isHyper 444 s2_misalign_stout.bits.paddr := s2_out.paddr 445 s2_misalign_stout.bits.gpaddr := s2_out.gpaddr 446 s2_misalign_stout.bits.isForVSnonLeafPTE := s2_out.isForVSnonLeafPTE 447 s2_misalign_stout.bits.need_rep := RegEnable(s1_tlb_miss, s1_fire) 448 s2_misalign_stout.bits.uop.exceptionVec := s2_out.uop.exceptionVec 449 io.misalign_stout := s2_misalign_stout 450 451 // mmio and exception 452 io.lsq_replenish := s2_out 453 io.lsq_replenish.af := s2_out.af && s2_valid && !s2_kill 454 455 // prefetch related 456 io.lsq_replenish.miss := io.dcache.resp.fire && io.dcache.resp.bits.miss // miss info 457 458 // RegNext prefetch train for better timing 459 // ** Now, prefetch train is valid at store s3 ** 460 val s2_prefetch_train_valid = WireInit(false.B) 461 s2_prefetch_train_valid := s2_valid && io.dcache.resp.fire && !s2_out.mmio && !s2_out.nc && !s2_in.tlbMiss && !s2_in.isHWPrefetch 462 if(EnableStorePrefetchSMS) { 463 io.s1_prefetch_spec := s1_fire 464 io.s2_prefetch_spec := s2_prefetch_train_valid 465 io.prefetch_train.valid := RegNext(s2_prefetch_train_valid) 466 io.prefetch_train.bits.fromLsPipelineBundle(s2_in, latch = true, enable = s2_prefetch_train_valid) 467 }else { 468 io.s1_prefetch_spec := false.B 469 io.s2_prefetch_spec := false.B 470 io.prefetch_train.valid := false.B 471 io.prefetch_train.bits.fromLsPipelineBundle(s2_in, latch = true, enable = false.B) 472 } 473 // override miss bit 474 io.prefetch_train.bits.miss := RegEnable(io.dcache.resp.bits.miss, s2_prefetch_train_valid) 475 // TODO: add prefetch and access bit 476 io.prefetch_train.bits.meta_prefetch := false.B 477 io.prefetch_train.bits.meta_access := false.B 478 479 // Pipeline 480 // -------------------------------------------------------------------------------- 481 // stage 3 482 // -------------------------------------------------------------------------------- 483 // store write back 484 val s3_valid = RegInit(false.B) 485 val s3_in = RegEnable(s2_out, s2_fire) 486 val s3_out = Wire(new MemExuOutput(isVector = true)) 487 val s3_kill = s3_in.uop.robIdx.needFlush(io.redirect) 488 val s3_can_go = s3_ready 489 val s3_fire = s3_valid && !s3_kill && s3_can_go 490 val s3_vecFeedback = RegEnable(s2_vecFeedback, s2_fire) 491 492 // store misalign will not writeback to rob now 493 when (s2_fire) { s3_valid := (!s2_mmio || s2_exception) && !s2_out.isHWPrefetch && !s2_mis_align && !s2_frm_mabuf } 494 .elsewhen (s3_fire) { s3_valid := false.B } 495 .elsewhen (s3_kill) { s3_valid := false.B } 496 497 // wb: writeback 498 val SelectGroupSize = RollbackGroupSize 499 val lgSelectGroupSize = log2Ceil(SelectGroupSize) 500 val TotalSelectCycles = scala.math.ceil(log2Ceil(LoadQueueRAWSize).toFloat / lgSelectGroupSize).toInt + 1 501 502 s3_out := DontCare 503 s3_out.uop := s3_in.uop 504 s3_out.data := DontCare 505 s3_out.debug.isMMIO := s3_in.mmio 506 s3_out.debug.isNC := s3_in.nc 507 s3_out.debug.paddr := s3_in.paddr 508 s3_out.debug.vaddr := s3_in.vaddr 509 s3_out.debug.isPerfCnt := false.B 510 511 // Pipeline 512 // -------------------------------------------------------------------------------- 513 // stage x 514 // -------------------------------------------------------------------------------- 515 // delay TotalSelectCycles - 2 cycle(s) 516 val TotalDelayCycles = TotalSelectCycles - 2 517 val sx_valid = Wire(Vec(TotalDelayCycles + 1, Bool())) 518 val sx_ready = Wire(Vec(TotalDelayCycles + 1, Bool())) 519 val sx_in = Wire(Vec(TotalDelayCycles + 1, new VecMemExuOutput(isVector = true))) 520 val sx_in_vec = Wire(Vec(TotalDelayCycles +1, Bool())) 521 522 // backward ready signal 523 s3_ready := sx_ready.head 524 for (i <- 0 until TotalDelayCycles + 1) { 525 if (i == 0) { 526 sx_valid(i) := s3_valid 527 sx_in(i).output := s3_out 528 sx_in(i).vecFeedback := s3_vecFeedback 529 sx_in(i).nc := s3_in.nc 530 sx_in(i).mmio := s3_in.mmio 531 sx_in(i).usSecondInv := s3_in.usSecondInv 532 sx_in(i).elemIdx := s3_in.elemIdx 533 sx_in(i).alignedType := s3_in.alignedType 534 sx_in(i).mbIndex := s3_in.mbIndex 535 sx_in(i).mask := s3_in.mask 536 sx_in(i).vaddr := s3_in.fullva 537 sx_in(i).vaNeedExt := s3_in.vaNeedExt 538 sx_in(i).gpaddr := s3_in.gpaddr 539 sx_in(i).isForVSnonLeafPTE := s3_in.isForVSnonLeafPTE 540 sx_in(i).vecTriggerMask := s3_in.vecTriggerMask 541 sx_in_vec(i) := s3_in.isvec 542 sx_ready(i) := !s3_valid(i) || sx_in(i).output.uop.robIdx.needFlush(io.redirect) || (if (TotalDelayCycles == 0) io.stout.ready else sx_ready(i+1)) 543 } else { 544 val cur_kill = sx_in(i).output.uop.robIdx.needFlush(io.redirect) 545 val cur_can_go = (if (i == TotalDelayCycles) io.stout.ready else sx_ready(i+1)) 546 val cur_fire = sx_valid(i) && !cur_kill && cur_can_go 547 val prev_fire = sx_valid(i-1) && !sx_in(i-1).output.uop.robIdx.needFlush(io.redirect) && sx_ready(i) 548 549 sx_ready(i) := !sx_valid(i) || cur_kill || (if (i == TotalDelayCycles) io.stout.ready else sx_ready(i+1)) 550 val sx_valid_can_go = prev_fire || cur_fire || cur_kill 551 sx_valid(i) := RegEnable(Mux(prev_fire, true.B, false.B), false.B, sx_valid_can_go) 552 sx_in(i) := RegEnable(sx_in(i-1), prev_fire) 553 sx_in_vec(i) := RegEnable(sx_in_vec(i-1), prev_fire) 554 } 555 } 556 val sx_last_valid = sx_valid.takeRight(1).head 557 val sx_last_ready = sx_ready.takeRight(1).head 558 val sx_last_in = sx_in.takeRight(1).head 559 val sx_last_in_vec = sx_in_vec.takeRight(1).head 560 sx_last_ready := !sx_last_valid || sx_last_in.output.uop.robIdx.needFlush(io.redirect) || io.stout.ready 561 562 // write back: normal store, nc store 563 io.stout.valid := sx_last_valid && !sx_last_in.output.uop.robIdx.needFlush(io.redirect) && !sx_last_in_vec //isStore(sx_last_in.output.uop.fuType) 564 io.stout.bits := sx_last_in.output 565 io.stout.bits.uop.exceptionVec := ExceptionNO.selectByFu(sx_last_in.output.uop.exceptionVec, StaCfg) 566 567 io.vecstout.valid := sx_last_valid && !sx_last_in.output.uop.robIdx.needFlush(io.redirect) && sx_last_in_vec //isVStore(sx_last_in.output.uop.fuType) 568 // TODO: implement it! 569 io.vecstout.bits.mBIndex := sx_last_in.mbIndex 570 io.vecstout.bits.hit := sx_last_in.vecFeedback 571 io.vecstout.bits.isvec := true.B 572 io.vecstout.bits.sourceType := RSFeedbackType.tlbMiss 573 io.vecstout.bits.flushState := DontCare 574 io.vecstout.bits.trigger := sx_last_in.output.uop.trigger 575 io.vecstout.bits.nc := sx_last_in.nc 576 io.vecstout.bits.mmio := sx_last_in.mmio 577 io.vecstout.bits.exceptionVec := ExceptionNO.selectByFu(sx_last_in.output.uop.exceptionVec, VstuCfg) 578 io.vecstout.bits.usSecondInv := sx_last_in.usSecondInv 579 io.vecstout.bits.vecFeedback := sx_last_in.vecFeedback 580 io.vecstout.bits.elemIdx := sx_last_in.elemIdx 581 io.vecstout.bits.alignedType := sx_last_in.alignedType 582 io.vecstout.bits.mask := sx_last_in.mask 583 io.vecstout.bits.vaddr := sx_last_in.vaddr 584 io.vecstout.bits.vaNeedExt := sx_last_in.vaNeedExt 585 io.vecstout.bits.gpaddr := sx_last_in.gpaddr 586 io.vecstout.bits.isForVSnonLeafPTE := sx_last_in.isForVSnonLeafPTE 587 io.vecstout.bits.vstart := sx_last_in.output.uop.vpu.vstart 588 io.vecstout.bits.vecTriggerMask := sx_last_in.vecTriggerMask 589 // io.vecstout.bits.reg_offset.map(_ := DontCare) 590 // io.vecstout.bits.elemIdx.map(_ := sx_last_in.elemIdx) 591 // io.vecstout.bits.elemIdxInsideVd.map(_ := DontCare) 592 // io.vecstout.bits.vecdata.map(_ := DontCare) 593 // io.vecstout.bits.mask.map(_ := DontCare) 594 // io.vecstout.bits.alignedType.map(_ := sx_last_in.alignedType) 595 596 io.debug_ls := DontCare 597 io.debug_ls.s1_robIdx := s1_in.uop.robIdx.value 598 io.debug_ls.s1_isTlbFirstMiss := io.tlb.resp.valid && io.tlb.resp.bits.miss && io.tlb.resp.bits.debug.isFirstIssue && !s1_in.isHWPrefetch 599 600 private def printPipeLine(pipeline: LsPipelineBundle, cond: Bool, name: String): Unit = { 601 XSDebug(cond, 602 p"$name" + p" pc ${Hexadecimal(pipeline.uop.pc)} " + 603 p"addr ${Hexadecimal(pipeline.vaddr)} -> ${Hexadecimal(pipeline.paddr)} " + 604 p"op ${Binary(pipeline.uop.fuOpType)} " + 605 p"data ${Hexadecimal(pipeline.data)} " + 606 p"mask ${Hexadecimal(pipeline.mask)}\n" 607 ) 608 } 609 610 printPipeLine(s0_out, s0_valid, "S0") 611 printPipeLine(s1_out, s1_valid, "S1") 612 613 // perf cnt 614 XSPerfAccumulate("s0_in_valid", s0_valid) 615 XSPerfAccumulate("s0_in_fire", s0_fire) 616 XSPerfAccumulate("s0_vecin_fire", s0_fire && s0_use_flow_vec) 617 XSPerfAccumulate("s0_in_fire_first_issue", s0_fire && s0_isFirstIssue) 618 XSPerfAccumulate("s0_addr_spec_success", s0_fire && !s0_use_flow_vec && s0_saddr(VAddrBits-1, 12) === s0_stin.src(0)(VAddrBits-1, 12)) 619 XSPerfAccumulate("s0_addr_spec_failed", s0_fire && !s0_use_flow_vec && s0_saddr(VAddrBits-1, 12) =/= s0_stin.src(0)(VAddrBits-1, 12)) 620 XSPerfAccumulate("s0_addr_spec_success_once", s0_fire && !s0_use_flow_vec && s0_saddr(VAddrBits-1, 12) === s0_stin.src(0)(VAddrBits-1, 12) && s0_isFirstIssue) 621 XSPerfAccumulate("s0_addr_spec_failed_once", s0_fire && !s0_use_flow_vec && s0_saddr(VAddrBits-1, 12) =/= s0_stin.src(0)(VAddrBits-1, 12) && s0_isFirstIssue) 622 623 XSPerfAccumulate("s1_in_valid", s1_valid) 624 XSPerfAccumulate("s1_in_fire", s1_fire) 625 XSPerfAccumulate("s1_in_fire_first_issue", s1_fire && s1_in.isFirstIssue) 626 XSPerfAccumulate("s1_tlb_miss", s1_fire && s1_tlb_miss) 627 XSPerfAccumulate("s1_tlb_miss_first_issue", s1_fire && s1_tlb_miss && s1_in.isFirstIssue) 628 // end 629} 630