1/*************************************************************************************** 2* Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences 3* Copyright (c) 2020-2021 Peng Cheng Laboratory 4* 5* XiangShan is licensed under Mulan PSL v2. 6* You can use this software according to the terms and conditions of the Mulan PSL v2. 7* You may obtain a copy of Mulan PSL v2 at: 8* http://license.coscl.org.cn/MulanPSL2 9* 10* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, 11* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, 12* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. 13* 14* See the Mulan PSL v2 for more details. 15***************************************************************************************/ 16 17package xiangshan.cache 18 19import chipsalliance.rocketchip.config.Parameters 20import chisel3._ 21import chisel3.util._ 22import freechips.rocketchip.tilelink.{ClientMetadata, TLClientParameters, TLEdgeOut} 23import utils.{Code, ParallelOR, ReplacementPolicy, SRAMTemplate, XSDebug, XSPerfAccumulate} 24import xiangshan.L1CacheErrorInfo 25 26import scala.math.max 27 28class L1BankedDataReadReq(implicit p: Parameters) extends DCacheBundle 29{ 30 val way_en = Bits(DCacheWays.W) 31 val addr = Bits(PAddrBits.W) 32} 33 34class L1BankedDataReadLineReq(implicit p: Parameters) extends L1BankedDataReadReq 35{ 36 val rmask = Bits(DCacheBanks.W) 37} 38 39// Now, we can write a cache-block in a single cycle 40class L1BankedDataWriteReq(implicit p: Parameters) extends L1BankedDataReadReq 41{ 42 val wmask = Bits(DCacheBanks.W) 43 val data = Vec(DCacheBanks, Bits(DCacheSRAMRowBits.W)) 44} 45 46class L1BankedDataReadResult(implicit p: Parameters) extends DCacheBundle 47{ 48 // you can choose which bank to read to save power 49 val ecc = Bits(eccBits.W) 50 val raw_data = Bits(DCacheSRAMRowBits.W) 51 52 def asECCData() = { 53 Cat(ecc, raw_data) 54 } 55} 56 57// Banked DCache Data 58// ----------------------------------------------------------------- 59// | Bank0 | Bank1 | Bank2 | Bank3 | Bank4 | Bank5 | Bank6 | Bank7 | 60// ----------------------------------------------------------------- 61// | Way0 | Way0 | Way0 | Way0 | Way0 | Way0 | Way0 | Way0 | 62// | Way1 | Way1 | Way1 | Way1 | Way1 | Way1 | Way1 | Way1 | 63// | .... | .... | .... | .... | .... | .... | .... | .... | 64// ----------------------------------------------------------------- 65abstract class AbstractBankedDataArray(implicit p: Parameters) extends DCacheModule 66{ 67 val io = IO(new DCacheBundle { 68 // load pipeline read word req 69 val read = Vec(LoadPipelineWidth, Flipped(DecoupledIO(new L1BankedDataReadReq))) 70 // main pipeline read / write line req 71 val readline = Flipped(DecoupledIO(new L1BankedDataReadLineReq)) 72 val write = Flipped(DecoupledIO(new L1BankedDataWriteReq)) 73 // data bank read resp (all banks) 74 val resp = Output(Vec(DCacheBanks, new L1BankedDataReadResult())) 75 // val nacks = Output(Vec(LoadPipelineWidth, Bool())) 76 val errors = Output(Vec(LoadPipelineWidth, new L1CacheErrorInfo)) 77 // when bank_conflict, read (1) port should be ignored 78 val bank_conflict_slow = Output(Vec(LoadPipelineWidth, Bool())) 79 val bank_conflict_fast = Output(Vec(LoadPipelineWidth, Bool())) 80 // customized cache op port 81 val cacheOp = Flipped(new DCacheInnerOpIO) 82 }) 83 assert(LoadPipelineWidth == 2) // BankedDataArray is designed for 2 port 84 85 def pipeMap[T <: Data](f: Int => T) = VecInit((0 until LoadPipelineWidth).map(f)) 86 87 def dumpRead() = { 88 (0 until LoadPipelineWidth) map { w => 89 when(io.read(w).valid) { 90 XSDebug(s"DataArray Read channel: $w valid way_en: %x addr: %x\n", 91 io.read(w).bits.way_en, io.read(w).bits.addr) 92 } 93 } 94 when(io.readline.valid) { 95 XSDebug(s"DataArray Read Line, valid way_en: %x addr: %x rmask %x\n", 96 io.readline.bits.way_en, io.readline.bits.addr, io.readline.bits.rmask) 97 } 98 } 99 100 def dumpWrite() = { 101 when(io.write.valid) { 102 XSDebug(s"DataArray Write valid way_en: %x addr: %x\n", 103 io.write.bits.way_en, io.write.bits.addr) 104 105 (0 until DCacheBanks) map { r => 106 XSDebug(s"cycle: $r data: %x wmask: %x\n", 107 io.write.bits.data(r), io.write.bits.wmask(r)) 108 } 109 } 110 } 111 112 def dumpResp() = { 113 XSDebug(s"DataArray ReadeResp channel:\n") 114 (0 until DCacheBanks) map { r => 115 XSDebug(s"cycle: $r data: %x\n", io.resp(r).raw_data) 116 } 117 } 118 119 def dump() = { 120 dumpRead 121 dumpWrite 122 dumpResp 123 } 124} 125 126class BankedDataArray(implicit p: Parameters) extends AbstractBankedDataArray { 127 def getECCFromEncWord(encWord: UInt) = { 128 require(encWord.getWidth == encWordBits) 129 encWord(encWordBits - 1, wordBits) 130 } 131 132 io.write.ready := true.B 133 134 // wrap data rows of 8 ways 135 class DataSRAMBank(index: Int) extends Module { 136 val io = IO(new Bundle() { 137 val w = new Bundle() { 138 val en = Input(Bool()) 139 val addr = Input(UInt()) 140 val way_en = Input(UInt(DCacheWays.W)) 141 val data = Input(UInt(DCacheSRAMRowBits.W)) 142 } 143 144 val r = new Bundle() { 145 val en = Input(Bool()) 146 val addr = Input(UInt()) 147 val way_en = Input(UInt(DCacheWays.W)) 148 val data = Output(UInt(DCacheSRAMRowBits.W)) 149 } 150 }) 151 152 val r_way_en_reg = RegNext(io.r.way_en) 153 154 // multiway data bank 155 val data_bank = Array.fill(DCacheWays) { 156 Module(new SRAMTemplate( 157 Bits(DCacheSRAMRowBits.W), 158 set = DCacheSets, 159 way = 1, 160 shouldReset = false, 161 holdRead = false, 162 singlePort = true 163 )) 164 } 165 166 for (w <- 0 until DCacheWays) { 167 val wen = io.w.en && io.w.way_en(w) 168 data_bank(w).io.w.req.valid := wen 169 data_bank(w).io.w.req.bits.apply( 170 setIdx = io.w.addr, 171 data = io.w.data, 172 waymask = 1.U 173 ) 174 data_bank(w).io.r.req.valid := io.r.en 175 data_bank(w).io.r.req.bits.apply(setIdx = io.r.addr) 176 } 177 178 val half = nWays / 2 179 val data_read = data_bank.map(_.io.r.resp.data(0)) 180 val data_left = Mux1H(r_way_en_reg.tail(half), data_read.take(half)) 181 val data_right = Mux1H(r_way_en_reg.head(half), data_read.drop(half)) 182 183 val sel_low = r_way_en_reg.tail(half).orR() 184 val row_data = Mux(sel_low, data_left, data_right) 185 186 io.r.data := row_data 187 188 def dump_r() = { 189 when(RegNext(io.r.en)) { 190 XSDebug("bank read addr %x way_en %x data %x\n", 191 RegNext(io.r.addr), 192 RegNext(io.r.way_en), 193 io.r.data 194 ) 195 } 196 } 197 198 def dump_w() = { 199 when(io.w.en) { 200 XSDebug("bank write addr %x way_en %x data %x\n", 201 io.w.addr, 202 io.w.way_en, 203 io.w.data 204 ) 205 } 206 } 207 208 def dump() = { 209 dump_w() 210 dump_r() 211 } 212 } 213 214 val data_banks = List.tabulate(DCacheBanks)(i => Module(new DataSRAMBank(i))) 215 val ecc_banks = List.fill(DCacheBanks)(Module(new SRAMTemplate( 216 Bits(eccBits.W), 217 set = DCacheSets, 218 way = DCacheWays, 219 shouldReset = false, 220 holdRead = false, 221 singlePort = true 222 ))) 223 224 data_banks.map(_.dump()) 225 226 val way_en = Wire(Vec(LoadPipelineWidth, io.read(0).bits.way_en.cloneType)) 227 val way_en_reg = RegNext(way_en) 228 val set_addrs = Wire(Vec(LoadPipelineWidth, UInt())) 229 val bank_addrs = Wire(Vec(LoadPipelineWidth, UInt())) 230 231 // read data_banks and ecc_banks 232 // for single port SRAM, do not allow read and write in the same cycle 233 val rwhazard = io.write.valid 234 val rrhazard = false.B // io.readline.valid 235 (0 until LoadPipelineWidth).map(rport_index => { 236 set_addrs(rport_index) := addr_to_dcache_set(io.read(rport_index).bits.addr) 237 bank_addrs(rport_index) := addr_to_dcache_bank(io.read(rport_index).bits.addr) 238 239 io.read(rport_index).ready := !(rwhazard || rrhazard) 240 241 // use way_en to select a way after data read out 242 assert(!(RegNext(io.read(rport_index).fire() && PopCount(io.read(rport_index).bits.way_en) > 1.U))) 243 way_en(rport_index) := io.read(rport_index).bits.way_en 244 }) 245 io.readline.ready := !(rwhazard) 246 247 // read each bank, get bank result 248 val bank_result = Wire(Vec(DCacheBanks, new L1BankedDataReadResult())) 249 dontTouch(bank_result) 250 val row_error = Wire(Vec(DCacheBanks, Bool())) 251 dontTouch(row_error) 252 val rr_bank_conflict = bank_addrs(0) === bank_addrs(1) && io.read(0).valid && io.read(1).valid 253 val rrl_bank_conflict_0 = io.read(0).valid && io.readline.valid && io.readline.bits.rmask(bank_addrs(0)) 254 val rrl_bank_conflict_1 = io.read(1).valid && io.readline.valid && io.readline.bits.rmask(bank_addrs(1)) 255 val rw_bank_conflict_0 = io.read(0).valid && rwhazard 256 val rw_bank_conflict_1 = io.read(1).valid && rwhazard 257 val perf_multi_read = io.read(0).valid && io.read(1).valid 258 io.bank_conflict_fast(0) := rw_bank_conflict_0 || rrl_bank_conflict_0 259 io.bank_conflict_slow(0) := RegNext(io.bank_conflict_fast(0)) 260 io.bank_conflict_fast(1) := rw_bank_conflict_1 || rrl_bank_conflict_1 || rr_bank_conflict 261 io.bank_conflict_slow(1) := RegNext(io.bank_conflict_fast(1)) 262 XSPerfAccumulate("data_array_multi_read", perf_multi_read) 263 XSPerfAccumulate("data_array_rr_bank_conflict", rr_bank_conflict) 264 XSPerfAccumulate("data_array_rrl_bank_conflict_0", rrl_bank_conflict_0) 265 XSPerfAccumulate("data_array_rrl_bank_conflict_1", rrl_bank_conflict_1) 266 XSPerfAccumulate("data_array_rw_bank_conflict_0", rw_bank_conflict_0) 267 XSPerfAccumulate("data_array_rw_bank_conflict_1", rw_bank_conflict_1) 268 XSPerfAccumulate("data_array_access_total", io.read(0).valid +& io.read(1).valid) 269 XSPerfAccumulate("data_array_read_0", io.read(0).valid) 270 XSPerfAccumulate("data_array_read_1", io.read(1).valid) 271 XSPerfAccumulate("data_array_read_line", io.readline.valid) 272 XSPerfAccumulate("data_array_write", io.write.valid) 273 274 for (bank_index <- 0 until DCacheBanks) { 275 // Set Addr & Read Way Mask 276 // 277 // Pipe 0 Pipe 1 278 // + + 279 // | | 280 // +------+-----------+-------+ 281 // X X 282 // X +------+ Bank Addr Match 283 // +---------+----------+ 284 // | 285 // +--------+--------+ 286 // | Data Bank | 287 // +-----------------+ 288 val bank_addr_matchs = WireInit(VecInit(List.tabulate(LoadPipelineWidth)(i => { 289 bank_addrs(i) === bank_index.U && io.read(i).valid 290 }))) 291 val readline_match = io.readline.valid && io.readline.bits.rmask(bank_index) 292 val bank_way_en = Mux(readline_match, 293 io.readline.bits.way_en, 294 Mux(bank_addr_matchs(0), way_en(0), way_en(1)) 295 ) 296 val bank_set_addr = Mux(readline_match, 297 addr_to_dcache_set(io.readline.bits.addr), 298 Mux(bank_addr_matchs(0), set_addrs(0), set_addrs(1)) 299 ) 300 301 // read raw data 302 val data_bank = data_banks(bank_index) 303 data_bank.io.r.en := bank_addr_matchs.asUInt.orR || readline_match 304 data_bank.io.r.way_en := bank_way_en 305 data_bank.io.r.addr := bank_set_addr 306 bank_result(bank_index).raw_data := data_bank.io.r.data 307 308 // read ECC 309 val ecc_bank = ecc_banks(bank_index) 310 ecc_bank.io.r.req.valid := bank_addr_matchs.asUInt.orR 311 ecc_bank.io.r.req.bits.apply(setIdx = bank_set_addr) 312 bank_result(bank_index).ecc := Mux1H(RegNext(bank_way_en), ecc_bank.io.r.resp.data) 313 314 // use ECC to check error 315 val data = bank_result(bank_index).asECCData() 316 row_error(bank_index) := dcacheParameters.dataCode.decode(data).error && RegNext(bank_addr_matchs.asUInt.orR) 317 } 318 319 // Select final read result 320 (0 until LoadPipelineWidth).map(rport_index => { 321 io.errors(rport_index).ecc_error.valid := RegNext(io.read(rport_index).fire()) && row_error.asUInt.orR() 322 io.errors(rport_index).ecc_error.bits := true.B 323 io.errors(rport_index).paddr.valid := io.errors(rport_index).ecc_error.valid 324 io.errors(rport_index).paddr.bits := RegNext(io.read(rport_index).bits.addr) 325 }) 326 io.resp := bank_result 327 328 // write data_banks & ecc_banks 329 val sram_waddr = addr_to_dcache_set(io.write.bits.addr) 330 for (bank_index <- 0 until DCacheBanks) { 331 // data write 332 val data_bank = data_banks(bank_index) 333 data_bank.io.w.en := io.write.valid && io.write.bits.wmask(bank_index) 334 data_bank.io.w.way_en := io.write.bits.way_en 335 data_bank.io.w.addr := sram_waddr 336 data_bank.io.w.data := io.write.bits.data(bank_index) 337 338 // ecc write 339 val ecc_bank = ecc_banks(bank_index) 340 ecc_bank.io.w.req.valid := io.write.valid && io.write.bits.wmask(bank_index) 341 ecc_bank.io.w.req.bits.apply( 342 setIdx = sram_waddr, 343 data = getECCFromEncWord(cacheParams.dataCode.encode((io.write.bits.data(bank_index)))), 344 waymask = io.write.bits.way_en 345 ) 346 when(ecc_bank.io.w.req.valid) { 347 XSDebug("write in ecc sram: bank %x set %x data %x waymask %x\n", 348 bank_index.U, 349 sram_waddr, 350 getECCFromEncWord(cacheParams.dataCode.encode((io.write.bits.data(bank_index)))), 351 io.write.bits.way_en 352 ); 353 } 354 } 355 356 // deal with customized cache op 357 require(nWays <= 32) 358 io.cacheOp.resp.bits := DontCare 359 val cacheOpShouldResp = WireInit(false.B) 360 when(io.cacheOp.req.valid){ 361 when( 362 CacheInstrucion.isReadData(io.cacheOp.req.bits.opCode) || 363 CacheInstrucion.isReadDataECC(io.cacheOp.req.bits.opCode) 364 ){ 365 for (bank_index <- 0 until DCacheBanks) { 366 val data_bank = data_banks(bank_index) 367 data_bank.io.r.en := true.B 368 data_bank.io.r.way_en := UIntToOH(io.cacheOp.req.bits.wayNum(4, 0)) 369 data_bank.io.r.addr := io.cacheOp.req.bits.index 370 } 371 cacheOpShouldResp := true.B 372 } 373 when(CacheInstrucion.isWriteData(io.cacheOp.req.bits.opCode)){ 374 for (bank_index <- 0 until DCacheBanks) { 375 val data_bank = data_banks(bank_index) 376 data_bank.io.w.en := true.B 377 data_bank.io.w.way_en := UIntToOH(io.cacheOp.req.bits.wayNum(4, 0)) 378 data_bank.io.w.addr := io.cacheOp.req.bits.index 379 data_bank.io.w.data := io.cacheOp.req.bits.write_data_vec(bank_index) 380 } 381 cacheOpShouldResp := true.B 382 } 383 when(CacheInstrucion.isWriteDataECC(io.cacheOp.req.bits.opCode)){ 384 for (bank_index <- 0 until DCacheBanks) { 385 val ecc_bank = ecc_banks(bank_index) 386 ecc_bank.io.w.req.valid := true.B 387 ecc_bank.io.w.req.bits.apply( 388 setIdx = io.cacheOp.req.bits.index, 389 data = io.cacheOp.req.bits.write_data_ecc, 390 waymask = UIntToOH(io.cacheOp.req.bits.wayNum(4, 0)) 391 ) 392 } 393 cacheOpShouldResp := true.B 394 } 395 } 396 io.cacheOp.resp.valid := RegNext(io.cacheOp.req.valid && cacheOpShouldResp) 397 for (bank_index <- 0 until DCacheBanks) { 398 io.cacheOp.resp.bits.read_data_vec(bank_index) := bank_result(bank_index).raw_data 399 } 400 io.cacheOp.resp.bits.read_data_ecc := Mux(io.cacheOp.resp.valid, 401 bank_result(io.cacheOp.req.bits.bank_num).ecc, 402 0.U 403 ) 404} 405