1/*************************************************************************************** 2* Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences 3* Copyright (c) 2020-2021 Peng Cheng Laboratory 4* 5* XiangShan is licensed under Mulan PSL v2. 6* You can use this software according to the terms and conditions of the Mulan PSL v2. 7* You may obtain a copy of Mulan PSL v2 at: 8* http://license.coscl.org.cn/MulanPSL2 9* 10* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, 11* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, 12* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. 13* 14* See the Mulan PSL v2 for more details. 15***************************************************************************************/ 16 17package xiangshan.cache 18 19import chipsalliance.rocketchip.config.Parameters 20import chisel3._ 21import chisel3.util._ 22import freechips.rocketchip.tilelink.{ClientMetadata, TLClientParameters, TLEdgeOut} 23import system.L1CacheErrorInfo 24import utils.{Code, ParallelOR, ReplacementPolicy, SRAMTemplate, XSDebug, XSPerfAccumulate} 25 26import scala.math.max 27 28class L1BankedDataReadReq(implicit p: Parameters) extends DCacheBundle 29{ 30 val way_en = Bits(DCacheWays.W) 31 val addr = Bits(PAddrBits.W) 32} 33 34class L1BankedDataReadLineReq(implicit p: Parameters) extends L1BankedDataReadReq 35{ 36 val rmask = Bits(DCacheBanks.W) 37} 38 39// Now, we can write a cache-block in a single cycle 40class L1BankedDataWriteReq(implicit p: Parameters) extends L1BankedDataReadReq 41{ 42 val wmask = Bits(DCacheBanks.W) 43 val data = Vec(DCacheBanks, Bits(DCacheSRAMRowBits.W)) 44} 45 46class L1BankedDataReadResult(implicit p: Parameters) extends DCacheBundle 47{ 48 // you can choose which bank to read to save power 49 val ecc = Bits(eccBits.W) 50 val raw_data = Bits(DCacheSRAMRowBits.W) 51 52 def asECCData() = { 53 Cat(ecc, raw_data) 54 } 55} 56 57// Banked DCache Data 58// ----------------------------------------------------------------- 59// | Bank0 | Bank1 | Bank2 | Bank3 | Bank4 | Bank5 | Bank6 | Bank7 | 60// ----------------------------------------------------------------- 61// | Way0 | Way0 | Way0 | Way0 | Way0 | Way0 | Way0 | Way0 | 62// | Way1 | Way1 | Way1 | Way1 | Way1 | Way1 | Way1 | Way1 | 63// | .... | .... | .... | .... | .... | .... | .... | .... | 64// ----------------------------------------------------------------- 65abstract class AbstractBankedDataArray(implicit p: Parameters) extends DCacheModule 66{ 67 val io = IO(new DCacheBundle { 68 // load pipeline read word req 69 val read = Vec(LoadPipelineWidth, Flipped(DecoupledIO(new L1BankedDataReadReq))) 70 // main pipeline read / write line req 71 val readline = Flipped(DecoupledIO(new L1BankedDataReadLineReq)) 72 val write = Flipped(DecoupledIO(new L1BankedDataWriteReq)) 73 // data bank read resp (all banks) 74 val resp = Output(Vec(DCacheBanks, new L1BankedDataReadResult())) 75 // val nacks = Output(Vec(LoadPipelineWidth, Bool())) 76 val errors = Output(Vec(LoadPipelineWidth, new L1CacheErrorInfo)) 77 // when bank_conflict, read (1) port should be ignored 78 val bank_conflict_slow = Output(Vec(LoadPipelineWidth, Bool())) 79 val bank_conflict_fast = Output(Vec(LoadPipelineWidth, Bool())) 80 }) 81 assert(LoadPipelineWidth == 2) // BankedDataArray is designed for 2 port 82 83 def pipeMap[T <: Data](f: Int => T) = VecInit((0 until LoadPipelineWidth).map(f)) 84 85 def dumpRead() = { 86 (0 until LoadPipelineWidth) map { w => 87 when(io.read(w).valid) { 88 XSDebug(s"DataArray Read channel: $w valid way_en: %x addr: %x\n", 89 io.read(w).bits.way_en, io.read(w).bits.addr) 90 } 91 } 92 when(io.readline.valid) { 93 XSDebug(s"DataArray Read Line, valid way_en: %x addr: %x rmask %x\n", 94 io.readline.bits.way_en, io.readline.bits.addr, io.readline.bits.rmask) 95 } 96 } 97 98 def dumpWrite() = { 99 when(io.write.valid) { 100 XSDebug(s"DataArray Write valid way_en: %x addr: %x\n", 101 io.write.bits.way_en, io.write.bits.addr) 102 103 (0 until DCacheBanks) map { r => 104 XSDebug(s"cycle: $r data: %x wmask: %x\n", 105 io.write.bits.data(r), io.write.bits.wmask(r)) 106 } 107 } 108 } 109 110 def dumpResp() = { 111 XSDebug(s"DataArray ReadeResp channel:\n") 112 (0 until DCacheBanks) map { r => 113 XSDebug(s"cycle: $r data: %x\n", io.resp(r).raw_data) 114 } 115 } 116 117 def dump() = { 118 dumpRead 119 dumpWrite 120 dumpResp 121 } 122} 123 124class BankedDataArray(implicit p: Parameters) extends AbstractBankedDataArray { 125 def getECCFromEncWord(encWord: UInt) = { 126 require(encWord.getWidth == encWordBits) 127 encWord(encWordBits - 1, wordBits) 128 } 129 130 io.write.ready := true.B 131 132 // wrap data rows of 8 ways 133 class DataSRAMBank(index: Int) extends Module { 134 val io = IO(new Bundle() { 135 val w = new Bundle() { 136 val en = Input(Bool()) 137 val addr = Input(UInt()) 138 val way_en = Input(UInt(DCacheWays.W)) 139 val data = Input(UInt(DCacheSRAMRowBits.W)) 140 } 141 142 val r = new Bundle() { 143 val en = Input(Bool()) 144 val addr = Input(UInt()) 145 val way_en = Input(UInt(DCacheWays.W)) 146 val data = Output(UInt(DCacheSRAMRowBits.W)) 147 } 148 }) 149 150 val r_way_en_reg = RegNext(io.r.way_en) 151 152 // multiway data bank 153 val data_bank = Array.fill(DCacheWays) { 154 Module(new SRAMTemplate( 155 Bits(DCacheSRAMRowBits.W), 156 set = DCacheSets, 157 way = 1, 158 shouldReset = false, 159 holdRead = false, 160 singlePort = true 161 )) 162 } 163 164 for (w <- 0 until DCacheWays) { 165 val wen = io.w.en && io.w.way_en(w) 166 data_bank(w).io.w.req.valid := wen 167 data_bank(w).io.w.req.bits.apply( 168 setIdx = io.w.addr, 169 data = io.w.data, 170 waymask = 1.U 171 ) 172 data_bank(w).io.r.req.valid := io.r.en 173 data_bank(w).io.r.req.bits.apply(setIdx = io.r.addr) 174 } 175 176 val half = nWays / 2 177 val data_read = data_bank.map(_.io.r.resp.data(0)) 178 val data_left = Mux1H(r_way_en_reg.tail(half), data_read.take(half)) 179 val data_right = Mux1H(r_way_en_reg.head(half), data_read.drop(half)) 180 181 val sel_low = r_way_en_reg.tail(half).orR() 182 val row_data = Mux(sel_low, data_left, data_right) 183 184 io.r.data := row_data 185 186 def dump_r() = { 187 when(RegNext(io.r.en)) { 188 XSDebug("bank read addr %x way_en %x data %x\n", 189 RegNext(io.r.addr), 190 RegNext(io.r.way_en), 191 io.r.data 192 ) 193 } 194 } 195 196 def dump_w() = { 197 when(io.w.en) { 198 XSDebug("bank write addr %x way_en %x data %x\n", 199 io.w.addr, 200 io.w.way_en, 201 io.w.data 202 ) 203 } 204 } 205 206 def dump() = { 207 dump_w() 208 dump_r() 209 } 210 } 211 212 val data_banks = List.tabulate(DCacheBanks)(i => Module(new DataSRAMBank(i))) 213 val ecc_banks = List.fill(DCacheBanks)(Module(new SRAMTemplate( 214 Bits(eccBits.W), 215 set = DCacheSets, 216 way = DCacheWays, 217 shouldReset = false, 218 holdRead = false, 219 singlePort = true 220 ))) 221 222 data_banks.map(_.dump()) 223 224 val way_en = Wire(Vec(LoadPipelineWidth, io.read(0).bits.way_en.cloneType)) 225 val way_en_reg = RegNext(way_en) 226 val set_addrs = Wire(Vec(LoadPipelineWidth, UInt())) 227 val bank_addrs = Wire(Vec(LoadPipelineWidth, UInt())) 228 229 // read data_banks and ecc_banks 230 // for single port SRAM, do not allow read and write in the same cycle 231 val rwhazard = io.write.valid 232 val rrhazard = false.B // io.readline.valid 233 (0 until LoadPipelineWidth).map(rport_index => { 234 set_addrs(rport_index) := addr_to_dcache_set(io.read(rport_index).bits.addr) 235 bank_addrs(rport_index) := addr_to_dcache_bank(io.read(rport_index).bits.addr) 236 237 io.read(rport_index).ready := !(rwhazard || rrhazard) 238 239 // use way_en to select a way after data read out 240 assert(!(RegNext(io.read(rport_index).fire() && PopCount(io.read(rport_index).bits.way_en) > 1.U))) 241 way_en(rport_index) := io.read(rport_index).bits.way_en 242 }) 243 io.readline.ready := !(rwhazard) 244 245 // read each bank, get bank result 246 val bank_result = Wire(Vec(DCacheBanks, new L1BankedDataReadResult())) 247 val row_error = Wire(Vec(DCacheBanks, Bool())) 248 val rr_bank_conflict = bank_addrs(0) === bank_addrs(1) && io.read(0).valid && io.read(1).valid 249 val rrl_bank_conflict_0 = io.read(0).valid && io.readline.valid && io.readline.bits.rmask(bank_addrs(0)) 250 val rrl_bank_conflict_1 = io.read(1).valid && io.readline.valid && io.readline.bits.rmask(bank_addrs(1)) 251 val rw_bank_conflict_0 = io.read(0).valid && rwhazard 252 val rw_bank_conflict_1 = io.read(1).valid && rwhazard 253 val perf_multi_read = io.read(0).valid && io.read(1).valid 254 io.bank_conflict_fast(0) := rw_bank_conflict_0 || rrl_bank_conflict_0 255 io.bank_conflict_slow(0) := RegNext(io.bank_conflict_fast(0)) 256 io.bank_conflict_fast(1) := rw_bank_conflict_1 || rrl_bank_conflict_1 || rr_bank_conflict 257 io.bank_conflict_slow(1) := RegNext(io.bank_conflict_fast(1)) 258 XSPerfAccumulate("data_array_multi_read", perf_multi_read) 259 XSPerfAccumulate("data_array_rr_bank_conflict", rr_bank_conflict) 260 XSPerfAccumulate("data_array_rrl_bank_conflict_0", rrl_bank_conflict_0) 261 XSPerfAccumulate("data_array_rrl_bank_conflict_1", rrl_bank_conflict_1) 262 XSPerfAccumulate("data_array_rw_bank_conflict_0", rw_bank_conflict_0) 263 XSPerfAccumulate("data_array_rw_bank_conflict_1", rw_bank_conflict_1) 264 XSPerfAccumulate("data_array_access_total", io.read(0).valid +& io.read(1).valid) 265 XSPerfAccumulate("data_array_read_0", io.read(0).valid) 266 XSPerfAccumulate("data_array_read_1", io.read(1).valid) 267 XSPerfAccumulate("data_array_read_line", io.readline.valid) 268 XSPerfAccumulate("data_array_write", io.write.valid) 269 270 for (bank_index <- 0 until DCacheBanks) { 271 // Set Addr & Read Way Mask 272 // 273 // Pipe 0 Pipe 1 274 // + + 275 // | | 276 // +------+-----------+-------+ 277 // X X 278 // X +------+ Bank Addr Match 279 // +---------+----------+ 280 // | 281 // +--------+--------+ 282 // | Data Bank | 283 // +-----------------+ 284 val bank_addr_matchs = WireInit(VecInit(List.tabulate(LoadPipelineWidth)(i => { 285 bank_addrs(i) === bank_index.U && io.read(i).valid 286 }))) 287 val readline_match = io.readline.valid && io.readline.bits.rmask(bank_index) 288 val bank_way_en = Mux(readline_match, 289 io.readline.bits.way_en, 290 Mux(bank_addr_matchs(0), way_en(0), way_en(1)) 291 ) 292 val bank_set_addr = Mux(readline_match, 293 addr_to_dcache_set(io.readline.bits.addr), 294 Mux(bank_addr_matchs(0), set_addrs(0), set_addrs(1)) 295 ) 296 297 // read raw data 298 val data_bank = data_banks(bank_index) 299 data_bank.io.r.en := bank_addr_matchs.asUInt.orR || readline_match 300 data_bank.io.r.way_en := bank_way_en 301 data_bank.io.r.addr := bank_set_addr 302 bank_result(bank_index).raw_data := data_bank.io.r.data 303 304 // read ECC 305 val ecc_bank = ecc_banks(bank_index) 306 ecc_bank.io.r.req.valid := bank_addr_matchs.asUInt.orR 307 ecc_bank.io.r.req.bits.apply(setIdx = bank_set_addr) 308 bank_result(bank_index).ecc := Mux1H(RegNext(bank_way_en), ecc_bank.io.r.resp.data) 309 310 // use ECC to check error 311 val data = bank_result(bank_index).asECCData() 312 row_error(bank_index) := dcacheParameters.dataCode.decode(data).error && RegNext(bank_addr_matchs.asUInt.orR) 313 } 314 315 // Select final read result 316 (0 until LoadPipelineWidth).map(rport_index => { 317 io.errors(rport_index).ecc_error.valid := RegNext(io.read(rport_index).fire()) && row_error.asUInt.orR() 318 io.errors(rport_index).ecc_error.bits := true.B 319 io.errors(rport_index).paddr.valid := io.errors(rport_index).ecc_error.valid 320 io.errors(rport_index).paddr.bits := RegNext(io.read(rport_index).bits.addr) 321 }) 322 io.resp := bank_result 323 324 // write data_banks & ecc_banks 325 val sram_waddr = addr_to_dcache_set(io.write.bits.addr) 326 for (bank_index <- 0 until DCacheBanks) { 327 // data write 328 val data_bank = data_banks(bank_index) 329 data_bank.io.w.en := io.write.valid && io.write.bits.wmask(bank_index) 330 data_bank.io.w.way_en := io.write.bits.way_en 331 data_bank.io.w.addr := sram_waddr 332 data_bank.io.w.data := io.write.bits.data(bank_index) 333 334 // ecc write 335 val ecc_bank = ecc_banks(bank_index) 336 ecc_bank.io.w.req.valid := io.write.valid && io.write.bits.wmask(bank_index) 337 ecc_bank.io.w.req.bits.apply( 338 setIdx = sram_waddr, 339 data = getECCFromEncWord(cacheParams.dataCode.encode((io.write.bits.data(bank_index)))), 340 waymask = io.write.bits.way_en 341 ) 342 when(ecc_bank.io.w.req.valid) { 343 XSDebug("write in ecc sram: bank %x set %x data %x waymask %x\n", 344 bank_index.U, 345 sram_waddr, 346 getECCFromEncWord(cacheParams.dataCode.encode((io.write.bits.data(bank_index)))), 347 io.write.bits.way_en 348 ); 349 } 350 } 351 352} 353