1/*************************************************************************************** 2* Copyright (c) 2024 Beijing Institute of Open Source Chip (BOSC) 3* Copyright (c) 2020-2024 Institute of Computing Technology, Chinese Academy of Sciences 4* Copyright (c) 2020-2021 Peng Cheng Laboratory 5* 6* XiangShan is licensed under Mulan PSL v2. 7* You can use this software according to the terms and conditions of the Mulan PSL v2. 8* You may obtain a copy of Mulan PSL v2 at: 9* http://license.coscl.org.cn/MulanPSL2 10* 11* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, 12* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, 13* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. 14* 15* See the Mulan PSL v2 for more details. 16***************************************************************************************/ 17 18package xiangshan.cache 19 20import org.chipsalliance.cde.config.Parameters 21import chisel3._ 22import utils._ 23import utility._ 24import chisel3.util._ 25import freechips.rocketchip.tilelink.{ClientMetadata, TLClientParameters, TLEdgeOut} 26import xiangshan.{L1CacheErrorInfo, XSCoreParamsKey} 27 28import scala.math.max 29 30class BankConflictDB(implicit p: Parameters) extends DCacheBundle{ 31 val addr = Vec(LoadPipelineWidth, Bits(PAddrBits.W)) 32 val set_index = Vec(LoadPipelineWidth, UInt((DCacheAboveIndexOffset - DCacheSetOffset).W)) 33 val bank_index = Vec(VLEN/DCacheSRAMRowBits, UInt((DCacheSetOffset - DCacheBankOffset).W)) 34 val way_index = UInt(wayBits.W) 35 val fake_rr_bank_conflict = Bool() 36} 37 38class L1BankedDataReadReq(implicit p: Parameters) extends DCacheBundle 39{ 40 val way_en = Bits(DCacheWays.W) 41 val addr = Bits(PAddrBits.W) 42} 43 44class L1BankedDataReadReqWithMask(implicit p: Parameters) extends DCacheBundle 45{ 46 val way_en = Bits(DCacheWays.W) 47 val addr = Bits(PAddrBits.W) 48 val bankMask = Bits(DCacheBanks.W) 49} 50 51class L1BankedDataReadLineReq(implicit p: Parameters) extends L1BankedDataReadReq 52{ 53 val rmask = Bits(DCacheBanks.W) 54} 55 56// Now, we can write a cache-block in a single cycle 57class L1BankedDataWriteReq(implicit p: Parameters) extends L1BankedDataReadReq 58{ 59 val wmask = Bits(DCacheBanks.W) 60 val data = Vec(DCacheBanks, Bits(DCacheSRAMRowBits.W)) 61} 62 63// cache-block write request without data 64class L1BankedDataWriteReqCtrl(implicit p: Parameters) extends L1BankedDataReadReq 65 66class L1BankedDataReadResult(implicit p: Parameters) extends DCacheBundle 67{ 68 // you can choose which bank to read to save power 69 val ecc = Bits(eccBits.W) 70 val raw_data = Bits(DCacheSRAMRowBits.W) 71 val error_delayed = Bool() // 1 cycle later than data resp 72 73 def asECCData() = { 74 Cat(ecc, raw_data) 75 } 76} 77 78class DataSRAMBankWriteReq(implicit p: Parameters) extends DCacheBundle { 79 val en = Bool() 80 val addr = UInt() 81 val way_en = UInt(DCacheWays.W) 82 val data = UInt(DCacheSRAMRowBits.W) 83} 84 85// wrap a sram 86class DataSRAM(bankIdx: Int, wayIdx: Int)(implicit p: Parameters) extends DCacheModule { 87 val io = IO(new Bundle() { 88 val w = new Bundle() { 89 val en = Input(Bool()) 90 val addr = Input(UInt()) 91 val data = Input(UInt(DCacheSRAMRowBits.W)) 92 } 93 94 val r = new Bundle() { 95 val en = Input(Bool()) 96 val addr = Input(UInt()) 97 val data = Output(UInt(DCacheSRAMRowBits.W)) 98 } 99 }) 100 101 // data sram 102 val data_sram = Module(new SRAMTemplate( 103 Bits(DCacheSRAMRowBits.W), 104 set = DCacheSets / DCacheSetDiv, 105 way = 1, 106 shouldReset = false, 107 holdRead = false, 108 singlePort = true 109 )) 110 111 data_sram.io.w.req.valid := io.w.en 112 data_sram.io.w.req.bits.apply( 113 setIdx = io.w.addr, 114 data = io.w.data, 115 waymask = 1.U 116 ) 117 data_sram.io.r.req.valid := io.r.en 118 data_sram.io.r.req.bits.apply(setIdx = io.r.addr) 119 io.r.data := data_sram.io.r.resp.data(0) 120 XSPerfAccumulate("part_data_read_counter", data_sram.io.r.req.valid) 121 122 def dump_r() = { 123 when(RegNext(io.r.en)) { 124 XSDebug("bank read set %x bank %x way %x data %x\n", 125 RegEnable(io.r.addr, io.r.en), 126 bankIdx.U, 127 wayIdx.U, 128 io.r.data 129 ) 130 } 131 } 132 133 def dump_w() = { 134 when(io.w.en) { 135 XSDebug("bank write set %x bank %x way %x data %x\n", 136 io.w.addr, 137 bankIdx.U, 138 wayIdx.U, 139 io.w.data 140 ) 141 } 142 } 143 144 def dump() = { 145 dump_w() 146 dump_r() 147 } 148} 149 150// wrap data rows of 8 ways 151class DataSRAMBank(index: Int)(implicit p: Parameters) extends DCacheModule { 152 val io = IO(new Bundle() { 153 val w = Input(new DataSRAMBankWriteReq) 154 155 val r = new Bundle() { 156 val en = Input(Bool()) 157 val addr = Input(UInt()) 158 val way_en = Input(UInt(DCacheWays.W)) 159 val data = Output(UInt(DCacheSRAMRowBits.W)) 160 } 161 }) 162 163 assert(RegNext(!io.w.en || PopCount(io.w.way_en) <= 1.U)) 164 assert(RegNext(!io.r.en || PopCount(io.r.way_en) <= 1.U)) 165 166 val r_way_en_reg = RegEnable(io.r.way_en, io.r.en) 167 168 // external controls do not read and write at the same time 169 val w_info = io.w 170 // val rw_bypass = RegNext(io.w.addr === io.r.addr && io.w.way_en === io.r.way_en && io.w.en) 171 172 // multiway data bank 173 val data_bank = Seq.fill(DCacheWays) { 174 Module(new SRAMTemplate( 175 Bits(DCacheSRAMRowBits.W), 176 set = DCacheSets / DCacheSetDiv, 177 way = 1, 178 shouldReset = false, 179 holdRead = false, 180 singlePort = true 181 )) 182 } 183 184 for (w <- 0 until DCacheWays) { 185 val wen = w_info.en && w_info.way_en(w) 186 data_bank(w).io.w.req.valid := wen 187 data_bank(w).io.w.req.bits.apply( 188 setIdx = w_info.addr, 189 data = w_info.data, 190 waymask = 1.U 191 ) 192 data_bank(w).io.r.req.valid := io.r.en 193 data_bank(w).io.r.req.bits.apply(setIdx = io.r.addr) 194 } 195 XSPerfAccumulate("part_data_read_counter", PopCount(Cat(data_bank.map(_.io.r.req.valid)))) 196 197 val half = nWays / 2 198 val data_read = data_bank.map(_.io.r.resp.data(0)) 199 val data_left = Mux1H(r_way_en_reg.tail(half), data_read.take(half)) 200 val data_right = Mux1H(r_way_en_reg.head(half), data_read.drop(half)) 201 202 val sel_low = r_way_en_reg.tail(half).orR 203 val row_data = Mux(sel_low, data_left, data_right) 204 205 io.r.data := row_data 206 207 def dump_r() = { 208 when(RegNext(io.r.en)) { 209 XSDebug("bank read addr %x way_en %x data %x\n", 210 RegEnable(io.r.addr, io.r.en), 211 RegEnable(io.r.way_en, io.r.en), 212 io.r.data 213 ) 214 } 215 } 216 217 def dump_w() = { 218 when(io.w.en) { 219 XSDebug("bank write addr %x way_en %x data %x\n", 220 io.w.addr, 221 io.w.way_en, 222 io.w.data 223 ) 224 } 225 } 226 227 def dump() = { 228 dump_w() 229 dump_r() 230 } 231} 232 233case object HasDataEccParam 234 235// Banked DCache Data 236// ----------------------------------------------------------------- 237// | Bank0 | Bank1 | Bank2 | Bank3 | Bank4 | Bank5 | Bank6 | Bank7 | 238// ----------------------------------------------------------------- 239// | Way0 | Way0 | Way0 | Way0 | Way0 | Way0 | Way0 | Way0 | 240// | Way1 | Way1 | Way1 | Way1 | Way1 | Way1 | Way1 | Way1 | 241// | .... | .... | .... | .... | .... | .... | .... | .... | 242// ----------------------------------------------------------------- 243abstract class AbstractBankedDataArray(implicit p: Parameters) extends DCacheModule 244{ 245 val DataEccParam = if(EnableDataEcc) Some(HasDataEccParam) else None 246 val ReadlinePortErrorIndex = LoadPipelineWidth 247 val io = IO(new DCacheBundle { 248 // load pipeline read word req 249 val read = Vec(LoadPipelineWidth, Flipped(DecoupledIO(new L1BankedDataReadReqWithMask))) 250 val is128Req = Input(Vec(LoadPipelineWidth, Bool())) 251 // main pipeline read / write line req 252 val readline_intend = Input(Bool()) 253 val readline = Flipped(DecoupledIO(new L1BankedDataReadLineReq)) 254 val write = Flipped(DecoupledIO(new L1BankedDataWriteReq)) 255 val write_dup = Vec(DCacheBanks, Flipped(Decoupled(new L1BankedDataWriteReqCtrl))) 256 // data for readline and loadpipe 257 val readline_resp = Output(Vec(DCacheBanks, new L1BankedDataReadResult())) 258 val readline_error_delayed = Output(Bool()) 259 val read_resp_delayed = Output(Vec(LoadPipelineWidth, Vec(VLEN/DCacheSRAMRowBits, new L1BankedDataReadResult()))) 260 val read_error_delayed = Output(Vec(LoadPipelineWidth,Vec(VLEN/DCacheSRAMRowBits, Bool()))) 261 // val nacks = Output(Vec(LoadPipelineWidth, Bool())) 262 // val errors = Output(Vec(LoadPipelineWidth + 1, ValidIO(new L1CacheErrorInfo))) // read ports + readline port 263 // when bank_conflict, read (1) port should be ignored 264 val bank_conflict_slow = Output(Vec(LoadPipelineWidth, Bool())) 265 val disable_ld_fast_wakeup = Output(Vec(LoadPipelineWidth, Bool())) 266 // customized cache op port 267 val cacheOp = Flipped(new L1CacheInnerOpIO) 268 val cacheOp_req_dup = Vec(DCacheDupNum, Flipped(Valid(new CacheCtrlReqInfo))) 269 val cacheOp_req_bits_opCode_dup = Input(Vec(DCacheDupNum, UInt(XLEN.W))) 270 }) 271 272 def pipeMap[T <: Data](f: Int => T) = VecInit((0 until LoadPipelineWidth).map(f)) 273 274 def getECCFromEncWord(encWord: UInt) = { 275 require(encWord.getWidth == encWordBits) 276 encWord(encWordBits - 1, wordBits) 277 } 278 279 def dumpRead = { 280 (0 until LoadPipelineWidth) map { w => 281 when(io.read(w).valid) { 282 XSDebug(s"DataArray Read channel: $w valid way_en: %x addr: %x\n", 283 io.read(w).bits.way_en, io.read(w).bits.addr) 284 } 285 } 286 when(io.readline.valid) { 287 XSDebug(s"DataArray Read Line, valid way_en: %x addr: %x rmask %x\n", 288 io.readline.bits.way_en, io.readline.bits.addr, io.readline.bits.rmask) 289 } 290 } 291 292 def dumpWrite = { 293 when(io.write.valid) { 294 XSDebug(s"DataArray Write valid way_en: %x addr: %x\n", 295 io.write.bits.way_en, io.write.bits.addr) 296 297 (0 until DCacheBanks) map { r => 298 XSDebug(s"cycle: $r data: %x wmask: %x\n", 299 io.write.bits.data(r), io.write.bits.wmask(r)) 300 } 301 } 302 } 303 304 def dumpResp = { 305 XSDebug(s"DataArray ReadeResp channel:\n") 306 (0 until LoadPipelineWidth) map { r => 307 XSDebug(s"cycle: $r data: %x\n", Mux(io.is128Req(r), 308 Cat(io.read_resp_delayed(r)(1).raw_data,io.read_resp_delayed(r)(0).raw_data), 309 io.read_resp_delayed(r)(0).raw_data)) 310 } 311 } 312 313 def dump() = { 314 dumpRead 315 dumpWrite 316 dumpResp 317 } 318} 319 320// the smallest access unit is sram 321class SramedDataArray(implicit p: Parameters) extends AbstractBankedDataArray { 322 println(" DCacheType: SramedDataArray") 323 val ReduceReadlineConflict = false 324 325 io.write.ready := true.B 326 io.write_dup.foreach(_.ready := true.B) 327 328 val data_banks = List.tabulate(DCacheSetDiv)( k => List.tabulate(DCacheBanks)(i => List.tabulate(DCacheWays)(j => Module(new DataSRAM(i,j))))) 329 // ecc_banks also needs to be changed to two-dimensional to align with data_banks 330 val ecc_banks = DataEccParam.map { 331 case _ => 332 val ecc = List.tabulate(DCacheSetDiv)( k => 333 List.tabulate(DCacheWays)(j => 334 List.tabulate(DCacheBanks)(i => 335 Module(new SRAMTemplate( 336 Bits(eccBits.W), 337 set = DCacheSets / DCacheSetDiv, 338 way = 1, 339 shouldReset = false, 340 holdRead = false, 341 singlePort = true 342 )) 343 ))) 344 ecc 345 } 346 347 data_banks.map(_.map(_.map(_.dump()))) 348 349 val way_en = Wire(Vec(LoadPipelineWidth, io.read(0).bits.way_en.cloneType)) 350 val set_addrs = Wire(Vec(LoadPipelineWidth, UInt())) 351 val div_addrs = Wire(Vec(LoadPipelineWidth, UInt())) 352 val bank_addrs = Wire(Vec(LoadPipelineWidth, Vec(VLEN/DCacheSRAMRowBits, UInt()))) 353 354 val line_set_addr = addr_to_dcache_div_set(io.readline.bits.addr) 355 val line_div_addr = addr_to_dcache_div(io.readline.bits.addr) 356 // when WPU is enabled, line_way_en is all enabled when read data 357 val line_way_en = Fill(DCacheWays, 1.U) // val line_way_en = io.readline.bits.way_en 358 val line_way_en_reg = RegEnable(io.readline.bits.way_en, 0.U(DCacheWays.W),io.readline.valid) 359 360 val write_bank_mask_reg = RegEnable(io.write.bits.wmask, 0.U(DCacheBanks.W), io.write.valid) 361 val write_data_reg = RegEnable(io.write.bits.data, io.write.valid) 362 val write_valid_reg = RegNext(io.write.valid) 363 val write_valid_dup_reg = io.write_dup.map(x => RegNext(x.valid)) 364 val write_wayen_dup_reg = io.write_dup.map(x => RegEnable(x.bits.way_en, 0.U(DCacheWays.W), x.valid)) 365 val write_set_addr_dup_reg = io.write_dup.map(x => RegEnable(addr_to_dcache_div_set(x.bits.addr), x.valid)) 366 val write_div_addr_dup_reg = io.write_dup.map(x => RegEnable(addr_to_dcache_div(x.bits.addr), x.valid)) 367 368 // read data_banks and ecc_banks 369 // for single port SRAM, do not allow read and write in the same cycle 370 val rrhazard = false.B // io.readline.valid 371 (0 until LoadPipelineWidth).map(rport_index => { 372 div_addrs(rport_index) := addr_to_dcache_div(io.read(rport_index).bits.addr) 373 set_addrs(rport_index) := addr_to_dcache_div_set(io.read(rport_index).bits.addr) 374 bank_addrs(rport_index)(0) := addr_to_dcache_bank(io.read(rport_index).bits.addr) 375 bank_addrs(rport_index)(1) := bank_addrs(rport_index)(0) + 1.U 376 377 // use way_en to select a way after data read out 378 assert(!(RegNext(io.read(rport_index).fire && PopCount(io.read(rport_index).bits.way_en) > 1.U))) 379 way_en(rport_index) := io.read(rport_index).bits.way_en 380 }) 381 382 // read conflict 383 val rr_bank_conflict = Seq.tabulate(LoadPipelineWidth)(x => Seq.tabulate(LoadPipelineWidth)(y => 384 io.read(x).valid && io.read(y).valid && 385 div_addrs(x) === div_addrs(y) && 386 (io.read(x).bits.bankMask & io.read(y).bits.bankMask) =/= 0.U && 387 io.read(x).bits.way_en === io.read(y).bits.way_en && 388 set_addrs(x) =/= set_addrs(y) 389 )) 390 val rrl_bank_conflict = Wire(Vec(LoadPipelineWidth, Bool())) 391 val rrl_bank_conflict_intend = Wire(Vec(LoadPipelineWidth, Bool())) 392 (0 until LoadPipelineWidth).foreach { i => 393 val judge = if (ReduceReadlineConflict) io.read(i).valid && (io.readline.bits.rmask & io.read(i).bits.bankMask) =/= 0.U && line_div_addr === div_addrs(i) && line_set_addr =/= set_addrs(i) 394 else io.read(i).valid && line_div_addr === div_addrs(i) && line_set_addr =/= set_addrs(i) 395 rrl_bank_conflict(i) := judge && io.readline.valid 396 rrl_bank_conflict_intend(i) := judge && io.readline_intend 397 } 398 val wr_bank_conflict = Seq.tabulate(LoadPipelineWidth)(x => 399 io.read(x).valid && write_valid_reg && 400 div_addrs(x) === write_div_addr_dup_reg.head && 401 way_en(x) === write_wayen_dup_reg.head && 402 (write_bank_mask_reg(bank_addrs(x)(0)) || write_bank_mask_reg(bank_addrs(x)(1)) && io.is128Req(x)) 403 ) 404 val wrl_bank_conflict = io.readline.valid && write_valid_reg && line_div_addr === write_div_addr_dup_reg.head 405 // ready 406 io.readline.ready := !(wrl_bank_conflict) 407 io.read.zipWithIndex.map { case (x, i) => x.ready := !(wr_bank_conflict(i) || rrhazard) } 408 409 val perf_multi_read = PopCount(io.read.map(_.valid)) >= 2.U 410 val bank_conflict_fast = Wire(Vec(LoadPipelineWidth, Bool())) 411 (0 until LoadPipelineWidth).foreach(i => { 412 bank_conflict_fast(i) := wr_bank_conflict(i) || rrl_bank_conflict(i) || 413 (if (i == 0) 0.B else (0 until i).map(rr_bank_conflict(_)(i)).reduce(_ || _)) 414 io.bank_conflict_slow(i) := RegNext(bank_conflict_fast(i)) 415 io.disable_ld_fast_wakeup(i) := wr_bank_conflict(i) || rrl_bank_conflict_intend(i) || 416 (if (i == 0) 0.B else (0 until i).map(rr_bank_conflict(_)(i)).reduce(_ || _)) 417 }) 418 XSPerfAccumulate("data_array_multi_read", perf_multi_read) 419 (1 until LoadPipelineWidth).foreach(y => (0 until y).foreach(x => 420 XSPerfAccumulate(s"data_array_rr_bank_conflict_${x}_${y}", rr_bank_conflict(x)(y)) 421 )) 422 (0 until LoadPipelineWidth).foreach(i => { 423 XSPerfAccumulate(s"data_array_rrl_bank_conflict_${i}", rrl_bank_conflict(i)) 424 XSPerfAccumulate(s"data_array_rw_bank_conflict_${i}", wr_bank_conflict(i)) 425 XSPerfAccumulate(s"data_array_read_${i}", io.read(i).valid) 426 }) 427 XSPerfAccumulate("data_array_access_total", PopCount(io.read.map(_.valid))) 428 XSPerfAccumulate("data_array_read_line", io.readline.valid) 429 XSPerfAccumulate("data_array_write", io.write.valid) 430 431 val read_result = Wire(Vec(DCacheSetDiv, Vec(DCacheBanks, Vec(DCacheWays,new L1BankedDataReadResult())))) 432 val read_result_delayed = Wire(Vec(DCacheSetDiv, Vec(DCacheBanks, Vec(DCacheWays,new L1BankedDataReadResult())))) 433 val read_error_delayed_result = Wire(Vec(DCacheSetDiv, Vec(DCacheBanks, Vec(DCacheWays, Bool())))) 434 dontTouch(read_result) 435 dontTouch(read_error_delayed_result) 436 for (div_index <- 0 until DCacheSetDiv){ 437 for (bank_index <- 0 until DCacheBanks) { 438 for (way_index <- 0 until DCacheWays) { 439 // Set Addr & Read Way Mask 440 // 441 // Pipe 0 .... Pipe (n-1) 442 // + .... + 443 // | .... | 444 // +----+---------------+-----+ 445 // X X 446 // X +------+ Bank Addr Match 447 // +---------+----------+ 448 // | 449 // +--------+--------+ 450 // | Data Bank | 451 // +-----------------+ 452 val loadpipe_en = WireInit(VecInit(List.tabulate(LoadPipelineWidth)(i => { 453 io.read(i).valid && div_addrs(i) === div_index.U && (bank_addrs(i)(0) === bank_index.U || bank_addrs(i)(1) === bank_index.U && io.is128Req(i)) && way_en(i)(way_index) 454 }))) 455 val readline_en = Wire(Bool()) 456 if (ReduceReadlineConflict) { 457 readline_en := io.readline.valid && io.readline.bits.rmask(bank_index) && line_way_en(way_index) && div_index.U === line_div_addr 458 } else { 459 readline_en := io.readline.valid && line_way_en(way_index) && div_index.U === line_div_addr 460 } 461 val sram_set_addr = Mux(readline_en, 462 addr_to_dcache_div_set(io.readline.bits.addr), 463 PriorityMux(Seq.tabulate(LoadPipelineWidth)(i => loadpipe_en(i) -> set_addrs(i))) 464 ) 465 val read_en = loadpipe_en.asUInt.orR || readline_en 466 // read raw data 467 val data_bank = data_banks(div_index)(bank_index)(way_index) 468 data_bank.io.r.en := read_en 469 data_bank.io.r.addr := sram_set_addr 470 ecc_banks match { 471 case Some(banks) => 472 val ecc_bank = banks(div_index)(bank_index)(way_index) 473 ecc_bank.io.r.req.valid := read_en 474 ecc_bank.io.r.req.bits.apply(setIdx = sram_set_addr) 475 read_result(div_index)(bank_index)(way_index).ecc := ecc_bank.io.r.resp.data(0) 476 case None => 477 read_result(div_index)(bank_index)(way_index).ecc := 0.U 478 } 479 480 read_result(div_index)(bank_index)(way_index).raw_data := data_bank.io.r.data 481 read_result_delayed(div_index)(bank_index)(way_index) := RegEnable(read_result(div_index)(bank_index)(way_index), RegNext(read_en)) 482 483 // use ECC to check error 484 ecc_banks match { 485 case Some(_) => 486 val ecc_data = read_result(div_index)(bank_index)(way_index).asECCData() 487 val ecc_data_delayed = RegEnable(ecc_data, RegNext(read_en)) 488 read_result(div_index)(bank_index)(way_index).error_delayed := dcacheParameters.dataCode.decode(ecc_data_delayed).error 489 read_error_delayed_result(div_index)(bank_index)(way_index) := read_result(div_index)(bank_index)(way_index).error_delayed 490 case None => 491 read_result(div_index)(bank_index)(way_index).error_delayed := false.B 492 read_error_delayed_result(div_index)(bank_index)(way_index) := false.B 493 } 494 } 495 } 496 } 497 498 val data_read_oh = WireInit(VecInit(Seq.fill(DCacheSetDiv * DCacheBanks * DCacheWays)(0.U(1.W)))) 499 for(div_index <- 0 until DCacheSetDiv){ 500 for (bank_index <- 0 until DCacheBanks) { 501 for (way_index <- 0 until DCacheWays) { 502 data_read_oh(div_index * DCacheBanks * DCacheWays + bank_index * DCacheWays + way_index) := data_banks(div_index)(bank_index)(way_index).io.r.en 503 } 504 } 505 } 506 XSPerfAccumulate("data_read_counter", PopCount(Cat(data_read_oh))) 507 508 // read result: expose banked read result 509 (0 until LoadPipelineWidth).map(i => { 510 // io.read_resp(i) := read_result(RegNext(bank_addrs(i)))(RegNext(OHToUInt(way_en(i)))) 511 val r_read_fire = RegNext(io.read(i).fire) 512 val rr_read_fire = RegNext(RegNext(io.read(i).fire)) 513 val rr_div_addr = RegEnable(RegEnable(div_addrs(i), io.read(i).fire), r_read_fire) 514 val rr_bank_addr = RegEnable(RegEnable(bank_addrs(i), io.read(i).fire), r_read_fire) 515 val rr_way_addr = RegEnable(RegEnable(OHToUInt(way_en(i)), io.read(i).fire), r_read_fire) 516 (0 until VLEN/DCacheSRAMRowBits).map( j =>{ 517 io.read_resp_delayed(i)(j) := read_result_delayed(rr_div_addr)(rr_bank_addr(j))(rr_way_addr) 518 // error detection 519 // normal read ports 520 io.read_error_delayed(i)(j) := rr_read_fire && read_error_delayed_result(rr_div_addr)(rr_bank_addr(j))(rr_way_addr) && !RegNext(io.bank_conflict_slow(i)) 521 }) 522 }) 523 524 // readline port 525 (0 until DCacheBanks).map(i => { 526 io.readline_resp(i) := read_result(RegEnable(line_div_addr, io.readline.valid))(i)(RegEnable(OHToUInt(io.readline.bits.way_en),io.readline.valid)) 527 }) 528 io.readline_error_delayed := RegNext(RegNext(io.readline.fire)) && 529 VecInit((0 until DCacheBanks).map(i => io.readline_resp(i).error_delayed)).asUInt.orR 530 531 // write data_banks & ecc_banks 532 for (div_index <- 0 until DCacheSetDiv) { 533 for (bank_index <- 0 until DCacheBanks) { 534 for (way_index <- 0 until DCacheWays) { 535 // data write 536 val wen_reg = write_bank_mask_reg(bank_index) && 537 write_valid_dup_reg(bank_index) && 538 write_div_addr_dup_reg(bank_index) === div_index.U && 539 write_wayen_dup_reg(bank_index)(way_index) 540 val data_bank = data_banks(div_index)(bank_index)(way_index) 541 data_bank.io.w.en := wen_reg 542 543 data_bank.io.w.addr := write_set_addr_dup_reg(bank_index) 544 data_bank.io.w.data := write_data_reg(bank_index) 545 // ecc write 546 ecc_banks match { 547 case Some(banks) => 548 val ecc_bank = banks(div_index)(bank_index)(way_index) 549 ecc_bank.io.w.req.valid := wen_reg 550 ecc_bank.io.w.req.bits.apply( 551 setIdx = write_set_addr_dup_reg(bank_index), 552 data = RegEnable(getECCFromEncWord(cacheParams.dataCode.encode((io.write.bits.data(bank_index)))), io.write.valid), 553 waymask = 1.U 554 ) 555 when(ecc_bank.io.w.req.valid) { 556 XSDebug("write in ecc sram: bank %x set %x data %x waymask %x\n", 557 bank_index.U, 558 addr_to_dcache_div_set(io.write.bits.addr), 559 getECCFromEncWord(cacheParams.dataCode.encode((io.write.bits.data(bank_index)))), 560 io.write.bits.way_en 561 ) 562 } 563 case None => None 564 } 565 } 566 } 567 } 568 569 require(nWays <= 32) 570 io.cacheOp.resp.bits := DontCare 571 val cacheOpShouldResp = WireInit(false.B) 572 val eccReadResult = Wire(Vec(DCacheBanks, UInt(eccBits.W))) 573 // DCacheDupNum is 16 574 // vec: the dupIdx for every bank and every group 575 val rdata_dup_vec = Seq(0,0,1,1,2,2,3,3) 576 val rdataEcc_dup_vec = Seq(4,4,5,5,6,6,7,7) 577 val wdata_dup_vec = Seq(8,8,9,9,10,10,11,11) 578 val wdataEcc_dup_vec = Seq(12,12,13,13,14,14,15,15) 579 val cacheOpDivAddr = set_to_dcache_div(io.cacheOp.req.bits.index) 580 val cacheOpSetAddr = set_to_dcache_div_set(io.cacheOp.req.bits.index) 581 val cacheOpWayNum = io.cacheOp.req.bits.wayNum(4, 0) 582 rdata_dup_vec.zipWithIndex.map{ case(dupIdx, bankIdx) => 583 for (divIdx <- 0 until DCacheSetDiv){ 584 for (wayIdx <- 0 until DCacheWays) { 585 when(io.cacheOp_req_dup(dupIdx).valid && CacheInstrucion.isReadData(io.cacheOp_req_bits_opCode_dup(dupIdx))) { 586 val data_bank = data_banks(divIdx)(bankIdx)(wayIdx) 587 data_bank.io.r.en := UIntToOH(io.cacheOp.req.bits.wayNum(4, 0))(wayIdx) && cacheOpDivAddr === divIdx.U 588 data_bank.io.r.addr := cacheOpSetAddr 589 cacheOpShouldResp := true.B 590 } 591 } 592 } 593 } 594 rdataEcc_dup_vec.zipWithIndex.map{ case(dupIdx, bankIdx) => 595 for (divIdx <- 0 until DCacheSetDiv) { 596 for (wayIdx <- 0 until DCacheWays) { 597 when(io.cacheOp_req_dup(dupIdx).valid && CacheInstrucion.isReadDataECC(io.cacheOp_req_bits_opCode_dup(dupIdx))) { 598 ecc_banks match { 599 case Some(banks) => 600 val ecc_bank = banks(divIdx)(bankIdx)(wayIdx) 601 ecc_bank.io.r.req.valid := true.B 602 ecc_bank.io.r.req.bits.setIdx := cacheOpSetAddr 603 cacheOpShouldResp := true.B 604 case None => 605 cacheOpShouldResp := true.B 606 } 607 } 608 } 609 } 610 } 611 wdata_dup_vec.zipWithIndex.map{ case(dupIdx, bankIdx) => 612 for (divIdx <- 0 until DCacheSetDiv) { 613 for (wayIdx <- 0 until DCacheWays) { 614 when(io.cacheOp_req_dup(dupIdx).valid && CacheInstrucion.isWriteData(io.cacheOp_req_bits_opCode_dup(dupIdx))) { 615 val data_bank = data_banks(divIdx)(bankIdx)(wayIdx) 616 data_bank.io.w.en := UIntToOH(io.cacheOp.req.bits.wayNum(4, 0))(wayIdx) && cacheOpDivAddr === divIdx.U 617 data_bank.io.w.addr := cacheOpSetAddr 618 data_bank.io.w.data := io.cacheOp.req.bits.write_data_vec(bankIdx) 619 cacheOpShouldResp := true.B 620 } 621 } 622 } 623 } 624 wdataEcc_dup_vec.zipWithIndex.map{ case(dupIdx, bankIdx) => 625 for (divIdx <- 0 until DCacheSetDiv) { 626 for (wayIdx <- 0 until DCacheWays) { 627 when(io.cacheOp_req_dup(dupIdx).valid && CacheInstrucion.isWriteDataECC(io.cacheOp_req_bits_opCode_dup(dupIdx))) { 628 ecc_banks match { 629 case Some(banks) => 630 val ecc_bank = banks(divIdx)(bankIdx)(wayIdx) 631 ecc_bank.io.w.req.valid := UIntToOH(io.cacheOp.req.bits.wayNum(4, 0))(wayIdx) && cacheOpDivAddr === divIdx.U 632 ecc_bank.io.w.req.bits.apply( 633 setIdx = cacheOpSetAddr, 634 data = io.cacheOp.req.bits.write_data_ecc, 635 waymask = 1.U 636 ) 637 cacheOpShouldResp := true.B 638 case None => 639 cacheOpShouldResp := true.B 640 } 641 } 642 } 643 } 644 } 645 io.cacheOp.resp.valid := RegNext(io.cacheOp.req.valid && cacheOpShouldResp) 646 for (bank_index <- 0 until DCacheBanks) { 647 val cacheOpDivAddrReg = RegEnable(cacheOpDivAddr, io.cacheOp.req.valid) 648 val cacheOpWayNumDivAddrReg = RegEnable(cacheOpWayNum, io.cacheOp.req.valid) 649 io.cacheOp.resp.bits.read_data_vec(bank_index) := read_result(cacheOpDivAddrReg)(bank_index)(cacheOpWayNumDivAddrReg).raw_data 650 eccReadResult(bank_index) := read_result(cacheOpDivAddrReg)(bank_index)(cacheOpWayNumDivAddrReg).ecc 651 } 652 653 io.cacheOp.resp.bits.read_data_ecc := Mux(io.cacheOp.resp.valid, 654 eccReadResult(RegEnable(io.cacheOp.req.bits.bank_num, io.cacheOp.req.valid)), 655 0.U 656 ) 657 658 val tableName = "BankConflict" + p(XSCoreParamsKey).HartId.toString 659 val siteName = "BankedDataArray" + p(XSCoreParamsKey).HartId.toString 660 val bankConflictTable = ChiselDB.createTable(tableName, new BankConflictDB) 661 val bankConflictData = Wire(new BankConflictDB) 662 for (i <- 0 until LoadPipelineWidth) { 663 bankConflictData.set_index(i) := set_addrs(i) 664 bankConflictData.addr(i) := io.read(i).bits.addr 665 } 666 667 // FIXME: rr_bank_conflict(0)(1) no generalization 668 when(rr_bank_conflict(0)(1)) { 669 (0 until (VLEN/DCacheSRAMRowBits)).map(i => { 670 bankConflictData.bank_index(i) := bank_addrs(0)(i) 671 }) 672 bankConflictData.way_index := OHToUInt(way_en(0)) 673 bankConflictData.fake_rr_bank_conflict := set_addrs(0) === set_addrs(1) && div_addrs(0) === div_addrs(1) 674 }.otherwise { 675 (0 until (VLEN/DCacheSRAMRowBits)).map(i => { 676 bankConflictData.bank_index(i) := 0.U 677 }) 678 bankConflictData.way_index := 0.U 679 bankConflictData.fake_rr_bank_conflict := false.B 680 } 681 682 val isWriteBankConflictTable = Constantin.createRecord(s"isWriteBankConflictTable${p(XSCoreParamsKey).HartId}") 683 bankConflictTable.log( 684 data = bankConflictData, 685 en = isWriteBankConflictTable.orR && rr_bank_conflict(0)(1), 686 site = siteName, 687 clock = clock, 688 reset = reset 689 ) 690 691 (1 until LoadPipelineWidth).foreach(y => (0 until y).foreach(x => 692 XSPerfAccumulate(s"data_array_fake_rr_bank_conflict_${x}_${y}", rr_bank_conflict(x)(y) && set_addrs(x)===set_addrs(y) && div_addrs(x) === div_addrs(y)) 693 )) 694 695} 696 697// the smallest access unit is bank 698class BankedDataArray(implicit p: Parameters) extends AbstractBankedDataArray { 699 println(" DCacheType: BankedDataArray") 700 val ReduceReadlineConflict = false 701 702 io.write.ready := true.B 703 io.write_dup.foreach(_.ready := true.B) 704 705 val data_banks = List.fill(DCacheSetDiv)(List.tabulate(DCacheBanks)(i => Module(new DataSRAMBank(i)))) 706 val ecc_banks = DataEccParam.map { 707 case _ => 708 val ecc = List.fill(DCacheSetDiv)(List.fill(DCacheBanks)( 709 Module(new SRAMTemplate( 710 Bits(eccBits.W), 711 set = DCacheSets / DCacheSetDiv, 712 way = DCacheWays, 713 shouldReset = false, 714 holdRead = false, 715 singlePort = true 716 )) 717 )) 718 ecc 719 } 720 721 data_banks.map(_.map(_.dump())) 722 723 val way_en = Wire(Vec(LoadPipelineWidth, io.read(0).bits.way_en.cloneType)) 724 val set_addrs = Wire(Vec(LoadPipelineWidth, UInt())) 725 val div_addrs = Wire(Vec(LoadPipelineWidth, UInt())) 726 val bank_addrs = Wire(Vec(LoadPipelineWidth, Vec(VLEN/DCacheSRAMRowBits, UInt()))) 727 val way_en_reg = Wire(Vec(LoadPipelineWidth, io.read(0).bits.way_en.cloneType)) 728 val set_addrs_reg = Wire(Vec(LoadPipelineWidth, UInt())) 729 730 val line_set_addr = addr_to_dcache_div_set(io.readline.bits.addr) 731 val line_div_addr = addr_to_dcache_div(io.readline.bits.addr) 732 val line_way_en = io.readline.bits.way_en 733 734 val write_bank_mask_reg = RegEnable(io.write.bits.wmask, io.write.valid) 735 val write_data_reg = RegEnable(io.write.bits.data, io.write.valid) 736 val write_valid_reg = RegNext(io.write.valid) 737 val write_valid_dup_reg = io.write_dup.map(x => RegNext(x.valid)) 738 val write_wayen_dup_reg = io.write_dup.map(x => RegEnable(x.bits.way_en, x.valid)) 739 val write_set_addr_dup_reg = io.write_dup.map(x => RegEnable(addr_to_dcache_div_set(x.bits.addr), x.valid)) 740 val write_div_addr_dup_reg = io.write_dup.map(x => RegEnable(addr_to_dcache_div(x.bits.addr), x.valid)) 741 742 // read data_banks and ecc_banks 743 // for single port SRAM, do not allow read and write in the same cycle 744 val rwhazard = RegNext(io.write.valid) 745 val rrhazard = false.B // io.readline.valid 746 (0 until LoadPipelineWidth).map(rport_index => { 747 div_addrs(rport_index) := addr_to_dcache_div(io.read(rport_index).bits.addr) 748 bank_addrs(rport_index)(0) := addr_to_dcache_bank(io.read(rport_index).bits.addr) 749 bank_addrs(rport_index)(1) := Mux(io.is128Req(rport_index), bank_addrs(rport_index)(0) + 1.U, DCacheBanks.asUInt) 750 set_addrs(rport_index) := addr_to_dcache_div_set(io.read(rport_index).bits.addr) 751 set_addrs_reg(rport_index) := RegEnable(addr_to_dcache_div_set(io.read(rport_index).bits.addr), io.read(rport_index).valid) 752 753 // use way_en to select a way after data read out 754 assert(!(RegNext(io.read(rport_index).fire && PopCount(io.read(rport_index).bits.way_en) > 1.U))) 755 way_en(rport_index) := io.read(rport_index).bits.way_en 756 way_en_reg(rport_index) := RegEnable(io.read(rport_index).bits.way_en, io.read(rport_index).valid) 757 }) 758 759 // read each bank, get bank result 760 val rr_bank_conflict = Seq.tabulate(LoadPipelineWidth)(x => Seq.tabulate(LoadPipelineWidth)(y => 761 io.read(x).valid && io.read(y).valid && 762 div_addrs(x) === div_addrs(y) && 763 (io.read(x).bits.bankMask & io.read(y).bits.bankMask) =/= 0.U 764 )) 765 val rrl_bank_conflict = Wire(Vec(LoadPipelineWidth, Bool())) 766 val rrl_bank_conflict_intend = Wire(Vec(LoadPipelineWidth, Bool())) 767 (0 until LoadPipelineWidth).foreach { i => 768 val judge = if (ReduceReadlineConflict) io.read(i).valid && (io.readline.bits.rmask & io.read(i).bits.bankMask) =/= 0.U && div_addrs(i) === line_div_addr 769 else io.read(i).valid && div_addrs(i)===line_div_addr 770 rrl_bank_conflict(i) := judge && io.readline.valid 771 rrl_bank_conflict_intend(i) := judge && io.readline_intend 772 } 773 val wr_bank_conflict = Seq.tabulate(LoadPipelineWidth)(x => 774 io.read(x).valid && 775 write_valid_reg && 776 div_addrs(x) === write_div_addr_dup_reg.head && 777 (write_bank_mask_reg(bank_addrs(x)(0)) || write_bank_mask_reg(bank_addrs(x)(1)) && io.is128Req(x)) 778 ) 779 val wrl_bank_conflict = io.readline.valid && write_valid_reg && line_div_addr === write_div_addr_dup_reg.head 780 // ready 781 io.readline.ready := !(wrl_bank_conflict) 782 io.read.zipWithIndex.map{case(x, i) => x.ready := !(wr_bank_conflict(i) || rrhazard)} 783 784 val perf_multi_read = PopCount(io.read.map(_.valid)) >= 2.U 785 (0 until LoadPipelineWidth).foreach(i => { 786 // remove fake rr_bank_conflict situation in s2 787 val real_other_bank_conflict_reg = RegNext(wr_bank_conflict(i) || rrl_bank_conflict(i)) 788 val real_rr_bank_conflict_reg = (if (i == 0) 0.B else (0 until i).map{ j => 789 RegNext(rr_bank_conflict(j)(i)) && 790 (way_en_reg(j) =/= way_en_reg(i) || set_addrs_reg(j) =/= set_addrs_reg(i)) 791 }.reduce(_ || _)) 792 io.bank_conflict_slow(i) := real_other_bank_conflict_reg || real_rr_bank_conflict_reg 793 794 // get result in s1 795 io.disable_ld_fast_wakeup(i) := wr_bank_conflict(i) || rrl_bank_conflict_intend(i) || 796 (if (i == 0) 0.B else (0 until i).map(rr_bank_conflict(_)(i)).reduce(_ || _)) 797 }) 798 XSPerfAccumulate("data_array_multi_read", perf_multi_read) 799 (1 until LoadPipelineWidth).foreach(y => (0 until y).foreach(x => 800 XSPerfAccumulate(s"data_array_rr_bank_conflict_${x}_${y}", rr_bank_conflict(x)(y)) 801 )) 802 (0 until LoadPipelineWidth).foreach(i => { 803 XSPerfAccumulate(s"data_array_rrl_bank_conflict_${i}", rrl_bank_conflict(i)) 804 XSPerfAccumulate(s"data_array_rw_bank_conflict_${i}", wr_bank_conflict(i)) 805 XSPerfAccumulate(s"data_array_read_${i}", io.read(i).valid) 806 }) 807 XSPerfAccumulate("data_array_access_total", PopCount(io.read.map(_.valid))) 808 XSPerfAccumulate("data_array_read_line", io.readline.valid) 809 XSPerfAccumulate("data_array_write", io.write.valid) 810 811 val bank_result = Wire(Vec(DCacheSetDiv, Vec(DCacheBanks, new L1BankedDataReadResult()))) 812 val bank_result_delayed = Wire(Vec(DCacheSetDiv, Vec(DCacheBanks, new L1BankedDataReadResult()))) 813 val ecc_result = Wire(Vec(DCacheSetDiv, Vec(DCacheBanks, Vec(DCacheWays, UInt(eccBits.W))))) 814 val read_bank_error_delayed = Wire(Vec(DCacheSetDiv, Vec(DCacheBanks, Bool()))) 815 dontTouch(bank_result) 816 dontTouch(read_bank_error_delayed) 817 for (div_index <- 0 until DCacheSetDiv) { 818 for (bank_index <- 0 until DCacheBanks) { 819 // Set Addr & Read Way Mask 820 // 821 // Pipe 0 .... Pipe (n-1) 822 // + .... + 823 // | .... | 824 // +----+---------------+-----+ 825 // X X 826 // X +------+ Bank Addr Match 827 // +---------+----------+ 828 // | 829 // +--------+--------+ 830 // | Data Bank | 831 // +-----------------+ 832 val bank_addr_matchs = WireInit(VecInit(List.tabulate(LoadPipelineWidth)(i => { 833 io.read(i).valid && div_addrs(i) === div_index.U && (bank_addrs(i)(0) === bank_index.U || bank_addrs(i)(1) === bank_index.U && io.is128Req(i)) 834 }))) 835 val readline_match = Wire(Bool()) 836 if (ReduceReadlineConflict) { 837 readline_match := io.readline.valid && io.readline.bits.rmask(bank_index) && line_div_addr === div_index.U 838 } else { 839 readline_match := io.readline.valid && line_div_addr === div_index.U 840 } 841 val bank_way_en = Mux(readline_match, 842 io.readline.bits.way_en, 843 PriorityMux(Seq.tabulate(LoadPipelineWidth)(i => bank_addr_matchs(i) -> way_en(i))) 844 ) 845 // it is too long of bank_way_en's caculation, so bank_way_en_reg can not be caculated by RegNext(bank_way_en) 846 val bank_way_en_reg = Mux(RegNext(readline_match), 847 RegEnable(io.readline.bits.way_en, io.readline.valid), 848 PriorityMux(Seq.tabulate(LoadPipelineWidth)(i => RegNext(bank_addr_matchs(i)) -> RegNext(way_en(i)))) 849 ) 850 val bank_set_addr = Mux(readline_match, 851 line_set_addr, 852 PriorityMux(Seq.tabulate(LoadPipelineWidth)(i => bank_addr_matchs(i) -> set_addrs(i))) 853 ) 854 855 val read_enable = bank_addr_matchs.asUInt.orR || readline_match 856 857 // read raw data 858 val data_bank = data_banks(div_index)(bank_index) 859 data_bank.io.r.en := read_enable 860 data_bank.io.r.way_en := bank_way_en 861 data_bank.io.r.addr := bank_set_addr 862 bank_result(div_index)(bank_index).raw_data := data_bank.io.r.data 863 bank_result_delayed(div_index)(bank_index) := RegEnable(bank_result(div_index)(bank_index), RegNext(read_enable)) 864 865 // read ECC 866 ecc_banks match { 867 case Some(banks) => 868 val ecc_bank = banks(div_index)(bank_index) 869 ecc_bank.io.r.req.valid := read_enable 870 ecc_bank.io.r.req.bits.apply(setIdx = bank_set_addr) 871 ecc_result(div_index)(bank_index) := ecc_bank.io.r.resp.data 872 bank_result(div_index)(bank_index).ecc := Mux1H(bank_way_en_reg, ecc_bank.io.r.resp.data) 873 case None => 874 ecc_result(div_index)(bank_index) := DontCare 875 bank_result(div_index)(bank_index).ecc := DontCare 876 } 877 878 // use ECC to check error 879 ecc_banks match { 880 case Some(_) => 881 val ecc_data = bank_result(div_index)(bank_index).asECCData() 882 val ecc_data_delayed = RegEnable(ecc_data, RegNext(read_enable)) 883 bank_result(div_index)(bank_index).error_delayed := dcacheParameters.dataCode.decode(ecc_data_delayed).error 884 read_bank_error_delayed(div_index)(bank_index) := bank_result(div_index)(bank_index).error_delayed 885 case None => 886 bank_result(div_index)(bank_index).error_delayed := false.B 887 read_bank_error_delayed(div_index)(bank_index) := false.B 888 } 889 } 890 } 891 892 val data_read_oh = WireInit(VecInit(Seq.fill(DCacheSetDiv)(0.U(XLEN.W)))) 893 for (div_index <- 0 until DCacheSetDiv){ 894 val temp = WireInit(VecInit(Seq.fill(DCacheBanks)(0.U(XLEN.W)))) 895 for (bank_index <- 0 until DCacheBanks) { 896 temp(bank_index) := PopCount(Fill(DCacheWays, data_banks(div_index)(bank_index).io.r.en.asUInt)) 897 } 898 data_read_oh(div_index) := temp.reduce(_ + _) 899 } 900 XSPerfAccumulate("data_read_counter", data_read_oh.foldLeft(0.U)(_ + _)) 901 902 (0 until LoadPipelineWidth).map(i => { 903 val r_read_fire = RegNext(io.read(i).fire) 904 val rr_read_fire = RegNext(r_read_fire) 905 val rr_div_addr = RegEnable(RegEnable(div_addrs(i), io.read(i).fire), r_read_fire) 906 val rr_bank_addr = RegEnable(RegEnable(bank_addrs(i), io.read(i).fire), r_read_fire) 907 val rr_way_addr = RegEnable(RegEnable(OHToUInt(way_en(i)), io.read(i).fire), r_read_fire) 908 (0 until VLEN/DCacheSRAMRowBits).map( j =>{ 909 io.read_resp_delayed(i)(j) := bank_result_delayed(rr_div_addr)(rr_bank_addr(j)) 910 // error detection 911 io.read_error_delayed(i)(j) := rr_read_fire && read_bank_error_delayed(rr_div_addr)(rr_bank_addr(j)) && !RegNext(io.bank_conflict_slow(i)) 912 }) 913 }) 914 915 // read result: expose banked read result 916 io.readline_resp := bank_result(RegEnable(line_div_addr, io.readline.valid)) 917 io.readline_error_delayed := RegNext(RegNext(io.readline.fire)) && 918 VecInit((0 until DCacheBanks).map(i => io.readline_resp(i).error_delayed)).asUInt.orR 919 920 // write data_banks & ecc_banks 921 for (div_index <- 0 until DCacheSetDiv) { 922 for (bank_index <- 0 until DCacheBanks) { 923 // data write 924 val wen_reg = write_bank_mask_reg(bank_index) && 925 write_valid_dup_reg(bank_index) && 926 write_div_addr_dup_reg(bank_index) === div_index.U && RegNext(io.write.valid) 927 val data_bank = data_banks(div_index)(bank_index) 928 data_bank.io.w.en := wen_reg 929 data_bank.io.w.way_en := write_wayen_dup_reg(bank_index) 930 data_bank.io.w.addr := write_set_addr_dup_reg(bank_index) 931 data_bank.io.w.data := write_data_reg(bank_index) 932 933 // ecc write 934 ecc_banks match { 935 case Some(banks) => 936 val ecc_bank = banks(div_index)(bank_index) 937 ecc_bank.io.w.req.valid := wen_reg 938 ecc_bank.io.w.req.bits.apply( 939 setIdx = write_set_addr_dup_reg(bank_index), 940 data = RegEnable(getECCFromEncWord(cacheParams.dataCode.encode((io.write.bits.data(bank_index)))), io.write.valid), 941 waymask = write_wayen_dup_reg(bank_index) 942 ) 943 when(ecc_bank.io.w.req.valid) { 944 XSDebug("write in ecc sram: bank %x set %x data %x waymask %x\n", 945 bank_index.U, 946 addr_to_dcache_div_set(io.write.bits.addr), 947 getECCFromEncWord(cacheParams.dataCode.encode((io.write.bits.data(bank_index)))), 948 io.write.bits.way_en 949 ) 950 } 951 case None => None 952 } 953 } 954 } 955 956 // deal with customized cache op 957 require(nWays <= 32) 958 io.cacheOp.resp.bits := DontCare 959 val cacheOpShouldResp = WireInit(false.B) 960 val eccReadResult = Wire(Vec(DCacheBanks, UInt(eccBits.W))) 961 // DCacheDupNum is 16 962 // vec: the dupIdx for every bank and every group 963 val rdata_dup_vec = Seq(0, 0, 1, 1, 2, 2, 3, 3) 964 val rdataEcc_dup_vec = Seq(4, 4, 5, 5, 6, 6, 7, 7) 965 val wdata_dup_vec = Seq(8, 8, 9, 9, 10, 10, 11, 11) 966 val wdataEcc_dup_vec = Seq(12, 12, 13, 13, 14, 14, 15, 15) 967 val cacheOpDivAddr = set_to_dcache_div(io.cacheOp.req.bits.index) 968 val cacheOpSetAddr = set_to_dcache_div_set(io.cacheOp.req.bits.index) 969 val cacheOpWayMask = UIntToOH(io.cacheOp.req.bits.wayNum(4, 0)) 970 rdata_dup_vec.zipWithIndex.map{ case(dupIdx, bankIdx) => 971 for (divIdx <- 0 until DCacheSetDiv) { 972 when(io.cacheOp_req_dup(dupIdx).valid && CacheInstrucion.isReadData(io.cacheOp_req_bits_opCode_dup(dupIdx))) { 973 val data_bank = data_banks(divIdx)(bankIdx) 974 data_bank.io.r.en := true.B 975 data_bank.io.r.way_en := cacheOpWayMask 976 data_bank.io.r.addr := cacheOpSetAddr 977 cacheOpShouldResp := true.B 978 } 979 } 980 } 981 rdataEcc_dup_vec.zipWithIndex.map{ case(dupIdx, bankIdx) => 982 for (divIdx <- 0 until DCacheSetDiv) { 983 when(io.cacheOp_req_dup(dupIdx).valid && CacheInstrucion.isReadDataECC(io.cacheOp_req_bits_opCode_dup(dupIdx))) { 984 ecc_banks match { 985 case Some(banks) => 986 val ecc_bank = banks(divIdx)(bankIdx) 987 ecc_bank.io.r.req.valid := true.B 988 ecc_bank.io.r.req.bits.setIdx := cacheOpSetAddr 989 cacheOpShouldResp := true.B 990 case None => 991 cacheOpShouldResp := true.B 992 } 993 } 994 } 995 } 996 wdata_dup_vec.zipWithIndex.map{ case(dupIdx, bankIdx) => 997 for (divIdx <- 0 until DCacheSetDiv) { 998 when(io.cacheOp_req_dup(dupIdx).valid && CacheInstrucion.isWriteData(io.cacheOp_req_bits_opCode_dup(dupIdx))) { 999 val data_bank = data_banks(divIdx)(bankIdx) 1000 data_bank.io.w.en := cacheOpDivAddr === divIdx.U 1001 data_bank.io.w.way_en := cacheOpWayMask 1002 data_bank.io.w.addr := cacheOpSetAddr 1003 data_bank.io.w.data := io.cacheOp.req.bits.write_data_vec(bankIdx) 1004 cacheOpShouldResp := true.B 1005 } 1006 } 1007 } 1008 wdataEcc_dup_vec.zipWithIndex.map{ case(dupIdx, bankIdx) => 1009 for (divIdx <- 0 until DCacheSetDiv) { 1010 when(io.cacheOp_req_dup(dupIdx).valid && CacheInstrucion.isWriteDataECC(io.cacheOp_req_bits_opCode_dup(dupIdx))) { 1011 ecc_banks match { 1012 case Some(banks) => 1013 val ecc_bank = banks(divIdx)(bankIdx) 1014 ecc_bank.io.w.req.valid := cacheOpDivAddr === divIdx.U 1015 ecc_bank.io.w.req.bits.apply( 1016 setIdx = cacheOpSetAddr, 1017 data = io.cacheOp.req.bits.write_data_ecc, 1018 waymask = cacheOpWayMask 1019 ) 1020 cacheOpShouldResp := true.B 1021 case None => 1022 cacheOpShouldResp := true.B 1023 } 1024 } 1025 } 1026 } 1027 1028 io.cacheOp.resp.valid := RegNext(io.cacheOp.req.valid && cacheOpShouldResp) 1029 for (bank_index <- 0 until DCacheBanks) { 1030 val cacheOpDivAddrReg = RegEnable(cacheOpDivAddr, io.cacheOp.req.valid) 1031 val cacheOpWayMaskReg = RegEnable(cacheOpWayMask, io.cacheOp.req.valid) 1032 io.cacheOp.resp.bits.read_data_vec(bank_index) := bank_result(cacheOpDivAddrReg)(bank_index).raw_data 1033 eccReadResult(bank_index) := Mux1H(cacheOpWayMaskReg, ecc_result(cacheOpDivAddrReg)(bank_index)) 1034 } 1035 1036 io.cacheOp.resp.bits.read_data_ecc := Mux(io.cacheOp.resp.valid, 1037 eccReadResult(RegEnable(io.cacheOp.req.bits.bank_num, io.cacheOp.req.valid)), 1038 0.U 1039 ) 1040 1041 val tableName = "BankConflict" + p(XSCoreParamsKey).HartId.toString 1042 val siteName = "BankedDataArray" + p(XSCoreParamsKey).HartId.toString 1043 val bankConflictTable = ChiselDB.createTable(tableName, new BankConflictDB) 1044 val bankConflictData = Wire(new BankConflictDB) 1045 for (i <- 0 until LoadPipelineWidth) { 1046 bankConflictData.set_index(i) := set_addrs(i) 1047 bankConflictData.addr(i) := io.read(i).bits.addr 1048 } 1049 1050 // FIXME: rr_bank_conflict(0)(1) no generalization 1051 when(rr_bank_conflict(0)(1)) { 1052 (0 until (VLEN/DCacheSRAMRowBits)).map(i => { 1053 bankConflictData.bank_index(i) := bank_addrs(0)(i) 1054 }) 1055 bankConflictData.way_index := OHToUInt(way_en(0)) 1056 bankConflictData.fake_rr_bank_conflict := set_addrs(0) === set_addrs(1) && div_addrs(0) === div_addrs(1) 1057 }.otherwise { 1058 (0 until (VLEN/DCacheSRAMRowBits)).map(i => { 1059 bankConflictData.bank_index(i) := 0.U 1060 }) 1061 bankConflictData.way_index := 0.U 1062 bankConflictData.fake_rr_bank_conflict := false.B 1063 } 1064 1065 val isWriteBankConflictTable = Constantin.createRecord(s"isWriteBankConflictTable${p(XSCoreParamsKey).HartId}") 1066 bankConflictTable.log( 1067 data = bankConflictData, 1068 en = isWriteBankConflictTable.orR && rr_bank_conflict(0)(1), 1069 site = siteName, 1070 clock = clock, 1071 reset = reset 1072 ) 1073 1074 (1 until LoadPipelineWidth).foreach(y => (0 until y).foreach(x => 1075 XSPerfAccumulate(s"data_array_fake_rr_bank_conflict_${x}_${y}", rr_bank_conflict(x)(y) && set_addrs(x) === set_addrs(y) && div_addrs(x) === div_addrs(y)) 1076 )) 1077 1078} 1079