1/*************************************************************************************** 2* Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences 3* Copyright (c) 2020-2021 Peng Cheng Laboratory 4* 5* XiangShan is licensed under Mulan PSL v2. 6* You can use this software according to the terms and conditions of the Mulan PSL v2. 7* You may obtain a copy of Mulan PSL v2 at: 8* http://license.coscl.org.cn/MulanPSL2 9* 10* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, 11* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, 12* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. 13* 14* See the Mulan PSL v2 for more details. 15***************************************************************************************/ 16 17package xiangshan.frontend 18 19import chipsalliance.rocketchip.config.Parameters 20import chisel3._ 21import chisel3.util._ 22import xiangshan._ 23import xiangshan.cache._ 24import xiangshan.cache.mmu._ 25import chisel3.experimental.verification 26import utils._ 27import xiangshan.backend.fu.{PMPReqBundle, PMPRespBundle} 28 29trait HasInstrMMIOConst extends HasXSParameter with HasIFUConst{ 30 def mmioBusWidth = 64 31 def mmioBusBytes = mmioBusWidth / 8 32 def maxInstrLen = 32 33} 34 35trait HasIFUConst extends HasXSParameter { 36 def align(pc: UInt, bytes: Int): UInt = Cat(pc(VAddrBits-1, log2Ceil(bytes)), 0.U(log2Ceil(bytes).W)) 37 // def groupAligned(pc: UInt) = align(pc, groupBytes) 38 // def packetAligned(pc: UInt) = align(pc, packetBytes) 39} 40 41class IfuToFtqIO(implicit p:Parameters) extends XSBundle { 42 val pdWb = Valid(new PredecodeWritebackBundle) 43} 44 45class FtqInterface(implicit p: Parameters) extends XSBundle { 46 val fromFtq = Flipped(new FtqToIfuIO) 47 val toFtq = new IfuToFtqIO 48} 49 50class UncacheInterface(implicit p: Parameters) extends XSBundle { 51 val fromUncache = Flipped(DecoupledIO(new InsUncacheResp)) 52 val toUncache = DecoupledIO( new InsUncacheReq ) 53} 54 55class ICacheInterface(implicit p: Parameters) extends XSBundle { 56 val toIMeta = Decoupled(new ICacheReadBundle) 57 val toIData = Decoupled(new ICacheReadBundle) 58 val toMissQueue = Vec(2,Decoupled(new ICacheMissReq)) 59 val fromIMeta = Input(new ICacheMetaRespBundle) 60 val fromIData = Input(new ICacheDataRespBundle) 61 val fromMissQueue = Vec(2,Flipped(Decoupled(new ICacheMissResp))) 62} 63 64class NewIFUIO(implicit p: Parameters) extends XSBundle { 65 val ftqInter = new FtqInterface 66 val icacheInter = new ICacheInterface 67 val toIbuffer = Decoupled(new FetchToIBuffer) 68 val iTLBInter = Vec(2, new BlockTlbRequestIO) 69 val uncacheInter = new UncacheInterface 70 val pmp = Vec(2, new Bundle { 71 val req = Valid(new PMPReqBundle()) 72 val resp = Flipped(new PMPRespBundle()) 73 }) 74 val rob_commits = Flipped(Vec(CommitWidth, Valid(new RobCommitInfo))) 75} 76 77// record the situation in which fallThruAddr falls into 78// the middle of an RVI inst 79class LastHalfInfo(implicit p: Parameters) extends XSBundle { 80 val valid = Bool() 81 val middlePC = UInt(VAddrBits.W) 82 def matchThisBlock(startAddr: UInt) = valid && middlePC === startAddr 83} 84 85class IfuToPreDecode(implicit p: Parameters) extends XSBundle { 86 val data = if(HasCExtension) Vec(PredictWidth + 1, UInt(16.W)) else Vec(PredictWidth, UInt(32.W)) 87 val startAddr = UInt(VAddrBits.W) 88 val fallThruAddr = UInt(VAddrBits.W) 89 val fallThruError = Bool() 90 val isDoubleLine = Bool() 91 val ftqOffset = Valid(UInt(log2Ceil(PredictWidth).W)) 92 val target = UInt(VAddrBits.W) 93 val pageFault = Vec(2, Bool()) 94 val accessFault = Vec(2, Bool()) 95 val instValid = Bool() 96 val lastHalfMatch = Bool() 97 val oversize = Bool() 98} 99 100class NewIFU(implicit p: Parameters) extends XSModule with HasICacheParameters 101{ 102 println(s"icache ways: ${nWays} sets:${nSets}") 103 val io = IO(new NewIFUIO) 104 val (toFtq, fromFtq) = (io.ftqInter.toFtq, io.ftqInter.fromFtq) 105 val (toMeta, toData, meta_resp, data_resp) = (io.icacheInter.toIMeta, io.icacheInter.toIData, io.icacheInter.fromIMeta, io.icacheInter.fromIData) 106 val (toMissQueue, fromMissQueue) = (io.icacheInter.toMissQueue, io.icacheInter.fromMissQueue) 107 val (toUncache, fromUncache) = (io.uncacheInter.toUncache , io.uncacheInter.fromUncache) 108 val (toITLB, fromITLB) = (VecInit(io.iTLBInter.map(_.req)), VecInit(io.iTLBInter.map(_.resp))) 109 val fromPMP = io.pmp.map(_.resp) 110 111 def isCrossLineReq(start: UInt, end: UInt): Bool = start(blockOffBits) ^ end(blockOffBits) 112 113 def isLastInCacheline(fallThruAddr: UInt): Bool = fallThruAddr(blockOffBits - 1, 1) === 0.U 114 115 def ResultHoldBypass[T<:Data](data: T, valid: Bool): T = { 116 Mux(valid, data, RegEnable(data, valid)) 117 } 118 119 //--------------------------------------------- 120 // Fetch Stage 1 : 121 // * Send req to ICache Meta/Data 122 // * Check whether need 2 line fetch 123 //--------------------------------------------- 124 125 val f0_valid = fromFtq.req.valid 126 val f0_ftq_req = fromFtq.req.bits 127 val f0_situation = VecInit(Seq(isCrossLineReq(f0_ftq_req.startAddr, f0_ftq_req.fallThruAddr), isLastInCacheline(f0_ftq_req.fallThruAddr))) 128 val f0_doubleLine = f0_situation(0) || f0_situation(1) 129 val f0_vSetIdx = VecInit(get_idx((f0_ftq_req.startAddr)), get_idx(f0_ftq_req.fallThruAddr)) 130 val f0_fire = fromFtq.req.fire() 131 132 val f0_flush, f1_flush, f2_flush, f3_flush = WireInit(false.B) 133 val from_bpu_f0_flush, from_bpu_f1_flush, from_bpu_f2_flush, from_bpu_f3_flush = WireInit(false.B) 134 135 from_bpu_f0_flush := fromFtq.flushFromBpu.shouldFlushByStage2(f0_ftq_req.ftqIdx) || 136 fromFtq.flushFromBpu.shouldFlushByStage3(f0_ftq_req.ftqIdx) 137 138 val f3_redirect = WireInit(false.B) 139 f3_flush := fromFtq.redirect.valid 140 f2_flush := f3_flush || f3_redirect 141 f1_flush := f2_flush || from_bpu_f1_flush 142 f0_flush := f1_flush || from_bpu_f0_flush 143 144 val f1_ready, f2_ready, f3_ready = WireInit(false.B) 145 146 //fetch: send addr to Meta/TLB and Data simultaneously 147 val fetch_req = List(toMeta, toData) 148 for(i <- 0 until 2) { 149 fetch_req(i).valid := f0_fire 150 fetch_req(i).bits.isDoubleLine := f0_doubleLine 151 fetch_req(i).bits.vSetIdx := f0_vSetIdx 152 } 153 154 fromFtq.req.ready := fetch_req(0).ready && fetch_req(1).ready && f1_ready && GTimer() > 500.U 155 156 XSPerfAccumulate("ifu_bubble_ftq_not_valid", !f0_valid ) 157 XSPerfAccumulate("ifu_bubble_pipe_stall", f0_valid && fetch_req(0).ready && fetch_req(1).ready && !f1_ready ) 158 XSPerfAccumulate("ifu_bubble_sram_0_busy", f0_valid && !fetch_req(0).ready ) 159 XSPerfAccumulate("ifu_bubble_sram_1_busy", f0_valid && !fetch_req(1).ready ) 160 161 //--------------------------------------------- 162 // Fetch Stage 2 : 163 // * Send req to ITLB and TLB Response (Get Paddr) 164 // * ICache Response (Get Meta and Data) 165 // * Hit Check (Generate hit signal and hit vector) 166 // * Get victim way 167 //--------------------------------------------- 168 169 //TODO: handle fetch exceptions 170 171 val tlbRespAllValid = WireInit(false.B) 172 173 val f1_valid = RegInit(false.B) 174 val f1_ftq_req = RegEnable(next = f0_ftq_req, enable=f0_fire) 175 val f1_situation = RegEnable(next = f0_situation, enable=f0_fire) 176 val f1_doubleLine = RegEnable(next = f0_doubleLine, enable=f0_fire) 177 val f1_vSetIdx = RegEnable(next = f0_vSetIdx, enable=f0_fire) 178 val f1_fire = f1_valid && tlbRespAllValid && f2_ready 179 180 f1_ready := f2_ready && tlbRespAllValid || !f1_valid 181 182 from_bpu_f1_flush := fromFtq.flushFromBpu.shouldFlushByStage3(f1_ftq_req.ftqIdx) 183 184 val preDecoder = Module(new PreDecode) 185 val (preDecoderIn, preDecoderOut) = (preDecoder.io.in, preDecoder.io.out) 186 187 //flush generate and to Ftq 188 val predecodeOutValid = WireInit(false.B) 189 190 when(f1_flush) {f1_valid := false.B} 191 .elsewhen(f0_fire && !f0_flush) {f1_valid := true.B} 192 .elsewhen(f1_fire) {f1_valid := false.B} 193 194 toITLB(0).valid := f1_valid 195 toITLB(0).bits.size := 3.U // TODO: fix the size 196 toITLB(0).bits.vaddr := f1_ftq_req.startAddr 197 toITLB(0).bits.debug.pc := f1_ftq_req.startAddr 198 199 toITLB(1).valid := f1_valid && f1_doubleLine 200 toITLB(1).bits.size := 3.U // TODO: fix the size 201 toITLB(1).bits.vaddr := f1_ftq_req.fallThruAddr 202 toITLB(1).bits.debug.pc := f1_ftq_req.fallThruAddr 203 204 toITLB.map{port => 205 port.bits.cmd := TlbCmd.exec 206 port.bits.robIdx := DontCare 207 port.bits.debug.isFirstIssue := DontCare 208 } 209 210 fromITLB.map(_.ready := true.B) 211 212 val (tlbRespValid, tlbRespPAddr) = (fromITLB.map(_.valid), VecInit(fromITLB.map(_.bits.paddr))) 213 val (tlbRespMiss) = (fromITLB.map(port => port.bits.miss && port.valid)) 214 val (tlbExcpPF, tlbExcpAF) = (fromITLB.map(port => port.bits.excp.pf.instr && port.valid), 215 fromITLB.map(port => (port.bits.excp.af.instr) && port.valid)) //TODO: Temp treat mmio req as access fault 216 217 tlbRespAllValid := tlbRespValid(0) && (tlbRespValid(1) || !f1_doubleLine) 218 219 val f1_pAddrs = tlbRespPAddr 220 val f1_pTags = VecInit(f1_pAddrs.map(get_phy_tag(_))) 221 222 val f1_tags = ResultHoldBypass(data = meta_resp.tags, valid = RegNext(toMeta.fire())) 223 val f1_cacheline_valid = ResultHoldBypass(data = meta_resp.valid, valid = RegNext(toMeta.fire())) 224 val f1_datas = ResultHoldBypass(data = data_resp.datas, valid = RegNext(toData.fire())) 225 226 val bank0_hit_vec = VecInit(f1_tags(0).zipWithIndex.map{ case(way_tag,i) => f1_cacheline_valid(0)(i) && way_tag === f1_pTags(0) }) 227 val bank1_hit_vec = VecInit(f1_tags(1).zipWithIndex.map{ case(way_tag,i) => f1_cacheline_valid(1)(i) && way_tag === f1_pTags(1) }) 228 val (bank0_hit,bank1_hit) = (ParallelOR(bank0_hit_vec) && !tlbExcpPF(0) && !tlbExcpAF(0), ParallelOR(bank1_hit_vec) && !tlbExcpPF(1) && !tlbExcpAF(1)) 229 val f1_hit = (bank0_hit && bank1_hit && f1_valid && f1_doubleLine) || (f1_valid && !f1_doubleLine && bank0_hit) 230 val f1_bank_hit_vec = VecInit(Seq(bank0_hit_vec, bank1_hit_vec)) 231 val f1_bank_hit = VecInit(Seq(bank0_hit, bank1_hit)) 232 233 234 val replacers = Seq.fill(2)(ReplacementPolicy.fromString(Some("random"),nWays,nSets/2)) 235 val f1_victim_masks = VecInit(replacers.zipWithIndex.map{case (replacer, i) => UIntToOH(replacer.way(f1_vSetIdx(i)))}) 236 237 val touch_sets = Seq.fill(2)(Wire(Vec(2, UInt(log2Ceil(nSets/2).W)))) 238 val touch_ways = Seq.fill(2)(Wire(Vec(2, Valid(UInt(log2Ceil(nWays).W)))) ) 239 240 ((replacers zip touch_sets) zip touch_ways).map{case ((r, s),w) => r.access(s,w)} 241 242 val f1_hit_data = VecInit(f1_datas.zipWithIndex.map { case(bank, i) => 243 val bank_hit_data = Mux1H(f1_bank_hit_vec(i).asUInt, bank) 244 bank_hit_data 245 }) 246 247 (0 until nWays).map{ w => 248 XSPerfAccumulate("line_0_hit_way_" + Integer.toString(w, 10), f1_fire && f1_bank_hit(0) && OHToUInt(f1_bank_hit_vec(0)) === w.U) 249 } 250 251 (0 until nWays).map{ w => 252 XSPerfAccumulate("line_0_victim_way_" + Integer.toString(w, 10), f1_fire && !f1_bank_hit(0) && OHToUInt(f1_victim_masks(0)) === w.U) 253 } 254 255 (0 until nWays).map{ w => 256 XSPerfAccumulate("line_1_hit_way_" + Integer.toString(w, 10), f1_fire && f1_doubleLine && f1_bank_hit(1) && OHToUInt(f1_bank_hit_vec(1)) === w.U) 257 } 258 259 (0 until nWays).map{ w => 260 XSPerfAccumulate("line_1_victim_way_" + Integer.toString(w, 10), f1_fire && f1_doubleLine && !f1_bank_hit(1) && OHToUInt(f1_victim_masks(1)) === w.U) 261 } 262 263 XSPerfAccumulate("ifu_bubble_f1_tlb_miss", f1_valid && !tlbRespAllValid ) 264 265 //--------------------------------------------- 266 // Fetch Stage 3 : 267 // * get data from last stage (hit from f1_hit_data/miss from missQueue response) 268 // * if at least one needed cacheline miss, wait for miss queue response (a wait_state machine) THIS IS TOO UGLY!!! 269 // * cut cacheline(s) and send to PreDecode 270 // * check if prediction is right (branch target and type, jump direction and type , jal target ) 271 //--------------------------------------------- 272 val f2_fetchFinish = Wire(Bool()) 273 274 val f2_valid = RegInit(false.B) 275 val f2_ftq_req = RegEnable(next = f1_ftq_req, enable = f1_fire) 276 val f2_situation = RegEnable(next = f1_situation, enable=f1_fire) 277 val f2_doubleLine = RegEnable(next = f1_doubleLine, enable=f1_fire) 278 val f2_fire = f2_valid && f2_fetchFinish && f3_ready 279 280 when(f2_flush) {f2_valid := false.B} 281 .elsewhen(f1_fire && !f1_flush) {f2_valid := true.B } 282 .elsewhen(f2_fire) {f2_valid := false.B} 283 284 val pmpExcpAF = fromPMP.map(port => port.instr) 285 val mmio = fromPMP.map(port => port.mmio) // TODO: handle it 286 287 288 val f2_pAddrs = RegEnable(next = f1_pAddrs, enable = f1_fire) 289 val f2_hit = RegEnable(next = f1_hit , enable = f1_fire) 290 val f2_bank_hit = RegEnable(next = f1_bank_hit, enable = f1_fire) 291 val f2_miss = f2_valid && !f2_hit 292 val (f2_vSetIdx, f2_pTags) = (RegEnable(next = f1_vSetIdx, enable = f1_fire), RegEnable(next = f1_pTags, enable = f1_fire)) 293 val f2_waymask = RegEnable(next = f1_victim_masks, enable = f1_fire) 294 //exception information 295 val f2_except_pf = RegEnable(next = VecInit(tlbExcpPF), enable = f1_fire) 296 val f2_except_af = VecInit(RegEnable(next = VecInit(tlbExcpAF), enable = f1_fire).zip(pmpExcpAF).map(a => a._1 || DataHoldBypass(a._2, RegNext(f1_fire)).asBool)) 297 val f2_except = VecInit((0 until 2).map{i => f2_except_pf(i) || f2_except_af(i)}) 298 val f2_has_except = f2_valid && (f2_except_af.reduce(_||_) || f2_except_pf.reduce(_||_)) 299 val f2_mmio = io.pmp(0).resp.mmio && !f2_except_af(0) && !f2_except_pf(0) && f2_valid 300 301 f2_ready := (f3_ready && f2_fetchFinish) || !f2_valid 302 303 304 io.pmp.zipWithIndex.map { case (p, i) => 305 p.req.valid := f2_fire 306 p.req.bits.addr := f2_pAddrs(i) 307 p.req.bits.size := 3.U // TODO 308 p.req.bits.cmd := TlbCmd.exec 309 } 310 311 //instruction 312 val wait_idle :: wait_queue_ready :: wait_send_req :: wait_two_resp :: wait_0_resp :: wait_1_resp :: wait_one_resp ::wait_finish ::Nil = Enum(8) 313 val wait_state = RegInit(wait_idle) 314 315 fromMissQueue.map{port => port.ready := true.B} 316 317 val (miss0_resp, miss1_resp) = (fromMissQueue(0).fire(), fromMissQueue(1).fire()) 318 val (bank0_fix, bank1_fix) = (miss0_resp && !f2_bank_hit(0), miss1_resp && f2_doubleLine && !f2_bank_hit(1)) 319 320 val only_0_miss = f2_valid && !f2_hit && !f2_doubleLine && !f2_has_except && !f2_mmio 321 val only_0_hit = f2_valid && f2_hit && !f2_doubleLine && !f2_mmio 322 val hit_0_hit_1 = f2_valid && f2_hit && f2_doubleLine && !f2_mmio 323 val (hit_0_miss_1 , miss_0_hit_1, miss_0_miss_1) = ( (f2_valid && !f2_bank_hit(1) && f2_bank_hit(0) && f2_doubleLine && !f2_has_except && !f2_mmio), 324 (f2_valid && !f2_bank_hit(0) && f2_bank_hit(1) && f2_doubleLine && !f2_has_except && !f2_mmio), 325 (f2_valid && !f2_bank_hit(0) && !f2_bank_hit(1) && f2_doubleLine && !f2_has_except && !f2_mmio), 326 ) 327 328 val hit_0_except_1 = f2_valid && f2_doubleLine && !f2_except(0) && f2_except(1) && f2_bank_hit(0) 329 val miss_0_except_1 = f2_valid && f2_doubleLine && !f2_except(0) && f2_except(1) && !f2_bank_hit(0) 330 //val fetch0_except_1 = hit_0_except_1 || miss_0_except_1 331 val except_0 = f2_valid && f2_except(0) 332 333 val f2_mq_datas = Reg(Vec(2, UInt(blockBits.W))) 334 335 when(fromMissQueue(0).fire) {f2_mq_datas(0) := fromMissQueue(0).bits.data} 336 when(fromMissQueue(1).fire) {f2_mq_datas(1) := fromMissQueue(1).bits.data} 337 338 switch(wait_state){ 339 is(wait_idle){ 340 when(miss_0_except_1){ 341 wait_state := Mux(toMissQueue(0).ready, wait_queue_ready ,wait_idle ) 342 }.elsewhen( only_0_miss || miss_0_hit_1){ 343 wait_state := Mux(toMissQueue(0).ready, wait_queue_ready ,wait_idle ) 344 }.elsewhen(hit_0_miss_1){ 345 wait_state := Mux(toMissQueue(1).ready, wait_queue_ready ,wait_idle ) 346 }.elsewhen( miss_0_miss_1 ){ 347 wait_state := Mux(toMissQueue(0).ready && toMissQueue(1).ready, wait_queue_ready ,wait_idle) 348 } 349 } 350 351 //TODO: naive logic for wait icache response 352 is(wait_queue_ready){ 353 wait_state := wait_send_req 354 } 355 356 is(wait_send_req) { 357 when(miss_0_except_1 || only_0_miss || hit_0_miss_1 || miss_0_hit_1){ 358 wait_state := wait_one_resp 359 }.elsewhen( miss_0_miss_1 ){ 360 wait_state := wait_two_resp 361 } 362 } 363 364 is(wait_one_resp) { 365 when( (miss_0_except_1 ||only_0_miss || miss_0_hit_1) && fromMissQueue(0).fire()){ 366 wait_state := wait_finish 367 }.elsewhen( hit_0_miss_1 && fromMissQueue(1).fire()){ 368 wait_state := wait_finish 369 } 370 } 371 372 is(wait_two_resp) { 373 when(fromMissQueue(0).fire() && fromMissQueue(1).fire()){ 374 wait_state := wait_finish 375 }.elsewhen( !fromMissQueue(0).fire() && fromMissQueue(1).fire() ){ 376 wait_state := wait_0_resp 377 }.elsewhen(fromMissQueue(0).fire() && !fromMissQueue(1).fire()){ 378 wait_state := wait_1_resp 379 } 380 } 381 382 is(wait_0_resp) { 383 when(fromMissQueue(0).fire()){ 384 wait_state := wait_finish 385 } 386 } 387 388 is(wait_1_resp) { 389 when(fromMissQueue(1).fire()){ 390 wait_state := wait_finish 391 } 392 } 393 394 is(wait_finish) { 395 when(f2_fire) {wait_state := wait_idle } 396 } 397 } 398 399 when(f2_flush) { wait_state := wait_idle } 400 401 (0 until 2).map { i => 402 if(i == 1) toMissQueue(i).valid := (hit_0_miss_1 || miss_0_miss_1) && wait_state === wait_queue_ready 403 else toMissQueue(i).valid := (only_0_miss || miss_0_hit_1 || miss_0_miss_1 || miss_0_except_1) && wait_state === wait_queue_ready 404 toMissQueue(i).bits.addr := f2_pAddrs(i) 405 toMissQueue(i).bits.vSetIdx := f2_vSetIdx(i) 406 toMissQueue(i).bits.waymask := f2_waymask(i) 407 toMissQueue(i).bits.clientID :=0.U 408 } 409 410 411 val miss_all_fix = (wait_state === wait_finish) 412 413 f2_fetchFinish := ((f2_valid && f2_hit) || (f2_valid && f2_mmio) || miss_all_fix || hit_0_except_1 || except_0) 414 415 XSPerfAccumulate("ifu_bubble_f2_miss", f2_valid && !f2_fetchFinish ) 416 417 (touch_ways zip touch_sets).zipWithIndex.map{ case((t_w,t_s), i) => 418 t_s(0) := f1_vSetIdx(i) 419 t_w(0).valid := f1_bank_hit(i) 420 t_w(0).bits := OHToUInt(f1_bank_hit_vec(i)) 421 422 t_s(1) := f2_vSetIdx(i) 423 t_w(1).valid := f2_valid && !f2_bank_hit(i) 424 t_w(1).bits := OHToUInt(f2_waymask(i)) 425 } 426 427 val sec_miss_reg = RegInit(0.U.asTypeOf(Vec(4, Bool()))) 428 val reservedRefillData = Reg(Vec(2, UInt(blockBits.W))) 429 val f2_hit_datas = RegEnable(next = f1_hit_data, enable = f1_fire) 430 val f2_datas = Wire(Vec(2, UInt(blockBits.W))) 431 432 f2_datas.zipWithIndex.map{case(bank,i) => 433 if(i == 0) bank := Mux(f2_bank_hit(i), f2_hit_datas(i),Mux(sec_miss_reg(2),reservedRefillData(1),Mux(sec_miss_reg(0),reservedRefillData(0), f2_mq_datas(i)))) 434 else bank := Mux(f2_bank_hit(i), f2_hit_datas(i),Mux(sec_miss_reg(3),reservedRefillData(1),Mux(sec_miss_reg(1),reservedRefillData(0), f2_mq_datas(i)))) 435 } 436 437 val f2_jump_valids = Fill(PredictWidth, !preDecoderOut.cfiOffset.valid) | Fill(PredictWidth, 1.U(1.W)) >> (~preDecoderOut.cfiOffset.bits) 438 val f2_predecode_valids = VecInit(preDecoderOut.pd.map(instr => instr.valid)).asUInt & f2_jump_valids 439 440 def cut(cacheline: UInt, start: UInt) : Vec[UInt] ={ 441 if(HasCExtension){ 442 val result = Wire(Vec(PredictWidth + 1, UInt(16.W))) 443 val dataVec = cacheline.asTypeOf(Vec(blockBytes * 2/ 2, UInt(16.W))) 444 val startPtr = Cat(0.U(1.W), start(blockOffBits-1, 1)) 445 (0 until PredictWidth + 1).foreach( i => 446 result(i) := dataVec(startPtr + i.U) 447 ) 448 result 449 } else { 450 val result = Wire(Vec(PredictWidth, UInt(32.W)) ) 451 val dataVec = cacheline.asTypeOf(Vec(blockBytes * 2/ 4, UInt(32.W))) 452 val startPtr = Cat(0.U(1.W), start(blockOffBits-1, 2)) 453 (0 until PredictWidth).foreach( i => 454 result(i) := dataVec(startPtr + i.U) 455 ) 456 result 457 } 458 } 459 460 val f2_cut_data = cut( Cat(f2_datas.map(cacheline => cacheline.asUInt ).reverse).asUInt, f2_ftq_req.startAddr ) 461 462 // deal with secondary miss in f1 463 val f2_0_f1_0 = ((f2_valid && !f2_bank_hit(0)) && f1_valid && (get_block_addr(f2_ftq_req.startAddr) === get_block_addr(f1_ftq_req.startAddr))) 464 val f2_0_f1_1 = ((f2_valid && !f2_bank_hit(0)) && f1_valid && f1_doubleLine && (get_block_addr(f2_ftq_req.startAddr) === get_block_addr(f1_ftq_req.startAddr + blockBytes.U))) 465 val f2_1_f1_0 = ((f2_valid && !f2_bank_hit(1) && f2_doubleLine) && f1_valid && (get_block_addr(f2_ftq_req.startAddr+ blockBytes.U) === get_block_addr(f1_ftq_req.startAddr) )) 466 val f2_1_f1_1 = ((f2_valid && !f2_bank_hit(1) && f2_doubleLine) && f1_valid && f1_doubleLine && (get_block_addr(f2_ftq_req.startAddr+ blockBytes.U) === get_block_addr(f1_ftq_req.startAddr + blockBytes.U) )) 467 468 val isSameLine = f2_0_f1_0 || f2_0_f1_1 || f2_1_f1_0 || f2_1_f1_1 469 val sec_miss_sit = VecInit(Seq(f2_0_f1_0, f2_0_f1_1, f2_1_f1_0, f2_1_f1_1)) 470 val hasSecMiss = RegInit(false.B) 471 472 when(f2_flush){ 473 sec_miss_reg.map(sig => sig := false.B) 474 hasSecMiss := false.B 475 }.elsewhen(isSameLine && !f1_flush && f2_fire){ 476 sec_miss_reg.zipWithIndex.map{case(sig, i) => sig := sec_miss_sit(i)} 477 hasSecMiss := true.B 478 }.elsewhen((!isSameLine || f1_flush) && hasSecMiss && f2_fire){ 479 sec_miss_reg.map(sig => sig := false.B) 480 hasSecMiss := false.B 481 } 482 483 when((f2_0_f1_0 || f2_0_f1_1) && f2_fire){ 484 reservedRefillData(0) := f2_mq_datas(0) 485 } 486 487 when((f2_1_f1_0 || f2_1_f1_1) && f2_fire){ 488 reservedRefillData(1) := f2_mq_datas(1) 489 } 490 491 492 //--------------------------------------------- 493 // Fetch Stage 4 : 494 // * get data from last stage (hit from f1_hit_data/miss from missQueue response) 495 // * if at least one needed cacheline miss, wait for miss queue response (a wait_state machine) THIS IS TOO UGLY!!! 496 // * cut cacheline(s) and send to PreDecode 497 // * check if prediction is right (branch target and type, jump direction and type , jal target ) 498 //--------------------------------------------- 499 val f3_valid = RegInit(false.B) 500 val f3_ftq_req = RegEnable(next = f2_ftq_req, enable=f2_fire) 501 val f3_situation = RegEnable(next = f2_situation, enable=f2_fire) 502 val f3_doubleLine = RegEnable(next = f2_doubleLine, enable=f2_fire) 503 504 val f3_cut_data = RegEnable(next = f2_cut_data, enable=f2_fire) 505 val f3_except_pf = RegEnable(next = f2_except_pf, enable = f2_fire) 506 val f3_except_af = RegEnable(next = f2_except_af, enable = f2_fire) 507 val f3_hit = RegEnable(next = f2_hit , enable = f2_fire) 508 val f3_mmio = RegEnable(next = f2_mmio , enable = f2_fire) 509 510 //assert((f3_ftq_req.startAddr + 34.U) >= f3_ftq_req.fallThruAddr, "Fall through address exceeds the limit") 511 512 val f3_lastHalf = RegInit(0.U.asTypeOf(new LastHalfInfo)) 513 val f3_lastHalfMatch = f3_lastHalf.matchThisBlock(f3_ftq_req.startAddr) 514 val f3_except = VecInit((0 until 2).map{i => f3_except_pf(i) || f3_except_af(i)}) 515 val f3_has_except = f3_valid && (f3_except_af.reduce(_||_) || f3_except_pf.reduce(_||_)) 516 val f3_pAddrs = RegEnable(next = f2_pAddrs, enable = f2_fire) 517 518 val f3_mmio_data = Reg(UInt(maxInstrLen.W)) 519 520 val f3_data = if(HasCExtension) Wire(Vec(PredictWidth + 1, UInt(16.W))) else Wire(Vec(PredictWidth, UInt(32.W))) 521 f3_data := f3_cut_data 522 523 //performance counter 524 val f3_only_0_hit = RegEnable(next = only_0_hit, enable = f2_fire) 525 val f3_only_0_miss = RegEnable(next = only_0_miss, enable = f2_fire) 526 val f3_hit_0_hit_1 = RegEnable(next = hit_0_hit_1, enable = f2_fire) 527 val f3_hit_0_miss_1 = RegEnable(next = hit_0_miss_1, enable = f2_fire) 528 val f3_miss_0_hit_1 = RegEnable(next = miss_0_hit_1, enable = f2_fire) 529 val f3_miss_0_miss_1 = RegEnable(next = miss_0_miss_1, enable = f2_fire) 530 531 val mmio_idle :: mmio_send_req :: mmio_w_resp :: mmio_resend :: mmio_resend_w_resp :: mmio_w_commit :: Nil = Enum(6) 532 val mmio_state = RegInit(mmio_idle) 533 534 val f3_req_is_mmio = f3_mmio && f3_valid 535 val mmio_has_commited = VecInit(io.rob_commits.map{commit => commit.valid && commit.bits.ftqIdx === f3_ftq_req.ftqIdx && commit.bits.ftqOffset === 0.U}).asUInt.orR 536 val f3_mmio_req_commit = f3_req_is_mmio && mmio_state === mmio_w_commit && mmio_has_commited 537 538 val f3_mmio_to_commit = f3_req_is_mmio && mmio_state === mmio_w_commit 539 val f3_mmio_to_commit_next = RegNext(f3_mmio_to_commit) 540 val f3_mmio_can_go = f3_mmio_to_commit && !f3_mmio_to_commit_next 541 542 val f3_ftq_flush_self = fromFtq.redirect.valid && RedirectLevel.flushItself(fromFtq.redirect.bits.level) 543 544 val f3_need_not_flush = f3_req_is_mmio && fromFtq.redirect.valid && !f3_ftq_flush_self 545 546 when(f3_flush && !f3_need_not_flush) {f3_valid := false.B} 547 .elsewhen(f2_fire && !f2_flush) {f3_valid := true.B } 548 .elsewhen(io.toIbuffer.fire() && !f3_req_is_mmio) {f3_valid := false.B} 549 .elsewhen{f3_req_is_mmio && f3_mmio_req_commit} {f3_valid := false.B} 550 551 val f3_mmio_use_seq_pc = RegInit(false.B) 552 553 val (redirect_ftqIdx, redirect_ftqOffset) = (fromFtq.redirect.bits.ftqIdx,fromFtq.redirect.bits.ftqOffset) 554 val redirect_mmio_req = fromFtq.redirect.valid && redirect_ftqIdx === f3_ftq_req.ftqIdx && redirect_ftqOffset === 0.U 555 556 when(RegNext(f2_fire && !f2_flush) && f3_req_is_mmio) { f3_mmio_use_seq_pc := true.B } 557 .elsewhen(redirect_mmio_req) { f3_mmio_use_seq_pc := false.B } 558 559 f3_ready := Mux(f3_req_is_mmio, io.toIbuffer.ready && f3_mmio_req_commit || !f3_valid , io.toIbuffer.ready || !f3_valid) 560 561 when(f3_req_is_mmio){ 562 f3_data(0) := f3_mmio_data(15, 0) 563 f3_data(1) := f3_mmio_data(31, 16) 564 } 565 566 when(fromUncache.fire()) {f3_mmio_data := fromUncache.bits.data} 567 568 569 switch(mmio_state){ 570 is(mmio_idle){ 571 when(f3_req_is_mmio){ 572 mmio_state := mmio_send_req 573 } 574 } 575 576 is(mmio_send_req){ 577 mmio_state := Mux(toUncache.fire(), mmio_w_resp, mmio_send_req ) 578 } 579 580 is(mmio_w_resp){ 581 when(fromUncache.fire()){ 582 val isRVC = fromUncache.bits.data(1,0) =/= 3.U 583 mmio_state := Mux(isRVC, mmio_resend , mmio_w_commit) 584 } 585 } 586 587 is(mmio_resend){ 588 mmio_state := Mux(toUncache.fire(), mmio_resend_w_resp, mmio_resend ) 589 } 590 591 is(mmio_resend_w_resp){ 592 when(fromUncache.fire()){ 593 mmio_state := mmio_w_commit 594 } 595 } 596 597 is(mmio_w_commit){ 598 when(mmio_has_commited){ 599 mmio_state := mmio_idle 600 } 601 } 602 } 603 604 when(f3_ftq_flush_self) { 605 mmio_state := mmio_idle 606 f3_mmio_data := 0.U 607 } 608 609 toUncache.valid := ((mmio_state === mmio_send_req) || (mmio_state === mmio_resend)) && f3_req_is_mmio 610 toUncache.bits.addr := Mux((mmio_state === mmio_resend), f3_pAddrs(0) + 2.U, f3_pAddrs(0)) 611 fromUncache.ready := true.B 612 613 val f3_bank_hit = RegEnable(next = f2_bank_hit, enable = f2_fire) 614 val f3_req_0 = io.toIbuffer.fire() 615 val f3_req_1 = io.toIbuffer.fire() && f3_doubleLine 616 val f3_hit_0 = io.toIbuffer.fire() & f3_bank_hit(0) 617 val f3_hit_1 = io.toIbuffer.fire() && f3_doubleLine & f3_bank_hit(1) 618 619 preDecoderIn.instValid := f3_valid && !f3_has_except 620 preDecoderIn.data := f3_data 621 preDecoderIn.startAddr := f3_ftq_req.startAddr 622 preDecoderIn.fallThruAddr := f3_ftq_req.fallThruAddr 623 preDecoderIn.fallThruError := f3_ftq_req.fallThruError 624 preDecoderIn.isDoubleLine := f3_doubleLine 625 preDecoderIn.ftqOffset := f3_ftq_req.ftqOffset 626 preDecoderIn.target := f3_ftq_req.target 627 preDecoderIn.oversize := f3_ftq_req.oversize 628 preDecoderIn.lastHalfMatch := f3_lastHalfMatch 629 preDecoderIn.pageFault := f3_except_pf 630 preDecoderIn.accessFault := f3_except_af 631 632 633 // TODO: What if next packet does not match? 634 when (f3_flush) { 635 f3_lastHalf.valid := false.B 636 }.elsewhen (io.toIbuffer.fire()) { 637 f3_lastHalf.valid := preDecoderOut.hasLastHalf 638 f3_lastHalf.middlePC := preDecoderOut.realEndPC 639 } 640 641 val f3_predecode_range = VecInit(preDecoderOut.pd.map(inst => inst.valid)).asUInt 642 val f3_mmio_range = VecInit((0 until PredictWidth).map(i => if(i ==0) true.B else false.B)) 643 644 io.toIbuffer.valid := f3_valid && (!f3_req_is_mmio || f3_mmio_can_go) 645 io.toIbuffer.bits.instrs := preDecoderOut.instrs 646 io.toIbuffer.bits.valid := Mux(f3_req_is_mmio, f3_mmio_range.asUInt, f3_predecode_range & preDecoderOut.instrRange.asUInt) 647 io.toIbuffer.bits.pd := preDecoderOut.pd 648 io.toIbuffer.bits.ftqPtr := f3_ftq_req.ftqIdx 649 io.toIbuffer.bits.pc := preDecoderOut.pc 650 io.toIbuffer.bits.ftqOffset.zipWithIndex.map{case(a, i) => a.bits := i.U; a.valid := preDecoderOut.takens(i) && !f3_req_is_mmio} 651 io.toIbuffer.bits.foldpc := preDecoderOut.pc.map(i => XORFold(i(VAddrBits-1,1), MemPredPCWidth)) 652 io.toIbuffer.bits.ipf := preDecoderOut.pageFault 653 io.toIbuffer.bits.acf := preDecoderOut.accessFault 654 io.toIbuffer.bits.crossPageIPFFix := preDecoderOut.crossPageIPF 655 656 //Write back to Ftq 657 val f3_cache_fetch = f3_valid && !(f2_fire && !f2_flush) 658 val finishFetchMaskReg = RegNext(f3_cache_fetch) 659 660 661 val f3_mmio_missOffset = Wire(ValidUndirectioned(UInt(log2Ceil(PredictWidth).W))) 662 f3_mmio_missOffset.valid := f3_req_is_mmio 663 f3_mmio_missOffset.bits := 0.U 664 665 toFtq.pdWb.valid := (!finishFetchMaskReg && f3_valid && !f3_req_is_mmio) || (f3_mmio_req_commit && f3_mmio_use_seq_pc) 666 toFtq.pdWb.bits.pc := preDecoderOut.pc 667 toFtq.pdWb.bits.pd := preDecoderOut.pd 668 toFtq.pdWb.bits.pd.zipWithIndex.map{case(instr,i) => instr.valid := Mux(f3_req_is_mmio, f3_mmio_range(i), f3_predecode_range(i))} 669 toFtq.pdWb.bits.ftqIdx := f3_ftq_req.ftqIdx 670 toFtq.pdWb.bits.ftqOffset := f3_ftq_req.ftqOffset.bits 671 toFtq.pdWb.bits.misOffset := Mux(f3_req_is_mmio, f3_mmio_missOffset, preDecoderOut.misOffset) 672 toFtq.pdWb.bits.cfiOffset := preDecoderOut.cfiOffset 673 toFtq.pdWb.bits.target := Mux(f3_req_is_mmio,Mux((f3_mmio_data(1,0) =/= 3.U), f3_ftq_req.startAddr + 2.U , f3_ftq_req.startAddr + 4.U) ,preDecoderOut.target) 674 toFtq.pdWb.bits.jalTarget := preDecoderOut.jalTarget 675 toFtq.pdWb.bits.instrRange := Mux(f3_req_is_mmio, f3_mmio_range, preDecoderOut.instrRange) 676 677 val predecodeFlush = preDecoderOut.misOffset.valid && f3_valid 678 val predecodeFlushReg = RegNext(predecodeFlush && !(f2_fire && !f2_flush)) 679 680 val perfinfo = IO(new Bundle(){ 681 val perfEvents = Output(new PerfEventsBundle(15)) 682 }) 683 684 val perfEvents = Seq( 685 ("frontendFlush ", f3_redirect ), 686 ("ifu_req ", io.toIbuffer.fire() ), 687 ("ifu_miss ", io.toIbuffer.fire() && !f3_hit ), 688 ("ifu_req_cacheline_0 ", f3_req_0 ), 689 ("ifu_req_cacheline_1 ", f3_req_1 ), 690 ("ifu_req_cacheline_0_hit ", f3_hit_1 ), 691 ("ifu_req_cacheline_1_hit ", f3_hit_1 ), 692 ("only_0_hit ", f3_only_0_hit && io.toIbuffer.fire() ), 693 ("only_0_miss ", f3_only_0_miss && io.toIbuffer.fire() ), 694 ("hit_0_hit_1 ", f3_hit_0_hit_1 && io.toIbuffer.fire() ), 695 ("hit_0_miss_1 ", f3_hit_0_miss_1 && io.toIbuffer.fire() ), 696 ("miss_0_hit_1 ", f3_miss_0_hit_1 && io.toIbuffer.fire() ), 697 ("miss_0_miss_1 ", f3_miss_0_miss_1 && io.toIbuffer.fire() ), 698 ("cross_line_block ", io.toIbuffer.fire() && f3_situation(0) ), 699 ("fall_through_is_cacheline_end", io.toIbuffer.fire() && f3_situation(1) ), 700 ) 701 702 for (((perf_out,(perf_name,perf)),i) <- perfinfo.perfEvents.perf_events.zip(perfEvents).zipWithIndex) { 703 perf_out.incr_step := RegNext(perf) 704 } 705 706 f3_redirect := (!predecodeFlushReg && predecodeFlush && !f3_req_is_mmio) || (f3_mmio_req_commit && f3_mmio_use_seq_pc) 707 708 XSPerfAccumulate("ifu_req", io.toIbuffer.fire() ) 709 XSPerfAccumulate("ifu_miss", io.toIbuffer.fire() && !f3_hit ) 710 XSPerfAccumulate("ifu_req_cacheline_0", f3_req_0 ) 711 XSPerfAccumulate("ifu_req_cacheline_1", f3_req_1 ) 712 XSPerfAccumulate("ifu_req_cacheline_0_hit", f3_hit_0 ) 713 XSPerfAccumulate("ifu_req_cacheline_1_hit", f3_hit_1 ) 714 XSPerfAccumulate("frontendFlush", f3_redirect ) 715 XSPerfAccumulate("only_0_hit", f3_only_0_hit && io.toIbuffer.fire() ) 716 XSPerfAccumulate("only_0_miss", f3_only_0_miss && io.toIbuffer.fire() ) 717 XSPerfAccumulate("hit_0_hit_1", f3_hit_0_hit_1 && io.toIbuffer.fire() ) 718 XSPerfAccumulate("hit_0_miss_1", f3_hit_0_miss_1 && io.toIbuffer.fire() ) 719 XSPerfAccumulate("miss_0_hit_1", f3_miss_0_hit_1 && io.toIbuffer.fire() ) 720 XSPerfAccumulate("miss_0_miss_1", f3_miss_0_miss_1 && io.toIbuffer.fire() ) 721 XSPerfAccumulate("cross_line_block", io.toIbuffer.fire() && f3_situation(0) ) 722 XSPerfAccumulate("fall_through_is_cacheline_end", io.toIbuffer.fire() && f3_situation(1) ) 723} 724