timing(MMIOBridge, TL2CHICoupledL2): optimize PCredit timing (#236)

This commit optimizes timing by:
* adding a pipeline stage to update `PCrdValids`
* adding a pipeline stage to arbitrate PCredits to all the slices
* always being ready for RXRSP responses

src/main/scala/coupledL2/tl2chi/MMIOBridge.scala

@@ -241,7 +241,6 @@ class MMIOBridgeImp(outer: MMIOBridge) extends LazyModuleImp(outer)
 
   val io = IO(new DecoupledNoSnpPortIO)
   val io_pCrd = IO(Vec(mmioBridgeSize, new PCrdQueryBundle))
-  val debug_pcrdGrantFire = IO(Output(Bool()))
 
   val entries = Seq.fill(mmioBridgeSize) { Module(new MMIOBridgeEntry(edge)) }
   val readys = VecInit(entries.map(_.io.req.ready))
@@ -293,11 +292,9 @@ class MMIOBridgeImp(outer: MMIOBridge) extends LazyModuleImp(outer)
   io.rx.dat.ready := Cat(entries.zipWithIndex.map { case (entry, i) =>
     entry.io.chi.rx.dat.ready && io.rx.dat.bits.txnID === i.U
   }).orR
-  io.rx.rsp.ready := Cat(entries.zipWithIndex.map { case (entry, i) =>
-    entry.io.chi.rx.rsp.ready && io.rx.rsp.bits.txnID === i.U
-  }).orR || isPCrdGrant
-
-  debug_pcrdGrantFire := io.rx.rsp.valid && isPCrdGrant
+  io.rx.rsp.ready := true.B
+  assert(!io.rx.rsp.valid || Cat(entries.zipWithIndex.map { case (entry, i) =>
+    entry.io.chi.rx.rsp.ready && io.rx.rsp.bits.txnID === i.U }).orR)
 
   dontTouch(io)
   dontTouch(bus)

src/main/scala/coupledL2/tl2chi/TL2CHICoupledL2.scala

@@ -167,7 +167,6 @@ class TL2CHICoupledL2(implicit p: Parameters) extends CoupledL2Base {
         val pCrdValids = RegInit(VecInit(Seq.fill(entries)(false.B)))
         val pCrdTypes = Reg(Vec(entries, UInt(PCRDTYPE_WIDTH.W)))
         val pCrdSrcIDs = Reg(Vec(entries, UInt(SRCID_WIDTH.W)))
-        val pCrdInsertOH = PriorityEncoderOH(pCrdValids.map(!_))
         val pCrdMatch = Wire(Vec(entries, Vec(entries, Bool())))
         val pCrdMatchEntryVec = pCrdMatch.map(_.asUInt.orR)
         val pCrdMatchEntryOH = PriorityEncoderOH(pCrdMatchEntryVec)
@@ -176,44 +175,65 @@ class TL2CHICoupledL2(implicit p: Parameters) extends CoupledL2Base {
           pCrdMatch.map(x => VecInit(PriorityEncoderOH(x)))
         )
 
+        // Insert a PCredit
+        val groupSize = 8
+        val ohs_s0 = Seq.tabulate(entries)(i => (BigInt(1) << i).asUInt(entries.W)) :+ 0.U(entries.W)
+        val avail_s0 = Wire(Vec(entries, Bool()))
+        val groups_s0 = (avail_s0 :+ true.B).zip(ohs_s0).grouped(groupSize).toList
+        val select_s0 = Wire(Vec(groups_s0.length, Bool()))
+        select_s0.zip(groups_s0).foreach { case (s, g) => s := g.map(_._1).reduce(_ || _) }
+        val select_s1 = RegInit(VecInit(Seq.fill(groups_s0.length)(false.B)))
+        when (isPCrdGrant) { select_s1 := select_s0 }
+        val ohs_s1 = RegInit(VecInit(Seq.fill(groups_s0.length)(0.U(entries.W))))
+        ohs_s1.zip(groups_s0).foreach { case (oh, g) => when (isPCrdGrant) { oh := ParallelPriorityMux(g) } }
+        val pCrdInsertOH_s1 = ParallelPriorityMux(select_s1, ohs_s1)
+        val isPCrdGrant_s1 = RegNext(isPCrdGrant)
+        val pCrdType_s1 = RegEnable(rxrsp.bits.pCrdType, isPCrdGrant)
+        val srcID_s1 = RegEnable(rxrsp.bits.srcID, isPCrdGrant)
         pCrdValids.zipWithIndex.foreach { case (v, i) =>
+          avail_s0(i) := !v && (!isPCrdGrant_s1 || !pCrdInsertOH_s1(i))
           val t = pCrdTypes(i)
           val srcID = pCrdSrcIDs(i)
-          val insert = pCrdInsertOH(i)
-          val free = pCrdFreeOH(i)
-          when (isPCrdGrant) {
-            when (insert) {
+          val insert_s1 = pCrdInsertOH_s1(i)
+          when (isPCrdGrant_s1) {
+            when (insert_s1) {
               v := true.B
-              t := rxrsp.bits.pCrdType
-              srcID := rxrsp.bits.srcID
+              t := pCrdType_s1
+              srcID := srcID_s1
             }
-            assert(!(v && insert), "P-Credit overflow")
+            assert(!(v && insert_s1), "P-Credit overflow")
           }
+        }
 
+        // Free a PCredit
+        pCrdValids.zipWithIndex.foreach { case (v, i) =>
+          val free = pCrdFreeOH(i)
           when (free) { v := false.B }
           assert(!free || v, "invalid entry should not be free")
         }
 
         for (i <- 0 until entries) {
           pCrdMatch(i) := VecInit(pCrdValids.zip(pCrdTypes).zip(pCrdSrcIDs).map { case ((v, t), s) =>
-            querys(i).valid && v &&
+            querys(i).valid && !grants(i) && v &&
             querys(i).bits.pCrdType === t &&
             querys(i).bits.srcID === s
           })
-          grants(i) := pCrdMatchEntryOH(i)
+          grants(i) := RegNext(pCrdMatchEntryOH(i))
         }
-
+        val grantCnt = RegInit(0.U(64.W))
+        grantCnt := grantCnt + PopCount(grants)
+        dontTouch(grantCnt)
 
         val rxrspSliceID = getSliceID(rxrsp.bits.txnID)
         slices.zipWithIndex.foreach { case (s, i) =>
-          s.io.out.rx.rsp.valid := rxrsp.valid && rxrspSliceID === i.U && !rxrspIsMMIO
+          s.io.out.rx.rsp.valid := rxrsp.valid && rxrspSliceID === i.U && !rxrspIsMMIO && !isPCrdGrant
           s.io.out.rx.rsp.bits := rxrsp.bits
           s.io.out.rx.rsp.bits.txnID := restoreTXNID(rxrsp.bits.txnID)
         }
-        mmio.io.rx.rsp.valid := rxrsp.valid && rxrspIsMMIO
+        mmio.io.rx.rsp.valid := rxrsp.valid && rxrspIsMMIO && !isPCrdGrant
         mmio.io.rx.rsp.bits := rxrsp.bits
         mmio.io.rx.rsp.bits.txnID := restoreTXNID(rxrsp.bits.txnID)
-        rxrsp.ready := Mux(
+        rxrsp.ready := rxrsp.bits.opcode === PCrdGrant || Mux(
           rxrspIsMMIO,
           mmio.io.rx.rsp.ready,
           Cat(slices.zipWithIndex.map { case (s, i) => s.io.out.rx.rsp.ready && rxrspSliceID === i.U }).orR

src/main/scala/coupledL2/tl2chi/chi/LinkLayer.scala

@@ -268,7 +268,7 @@ object Decoupled2LCredit {
   }
 }
 
-class LinkMonitor(implicit p: Parameters) extends L2Module with HasCHIMsgParameters {
+class LinkMonitor(implicit p: Parameters) extends L2Module with HasCHIOpcodes {
   val io = IO(new Bundle() {
     val in = Flipped(new DecoupledPortIO())
     val out = new PortIO
@@ -304,7 +304,24 @@ class LinkMonitor(implicit p: Parameters) extends L2Module with HasCHIMsgParamet
 
   io.out.syscoreq := true.B
 
+  val retryAckCnt = RegInit(0.U(64.W))
+  val pCrdGrantCnt = RegInit(0.U(64.W))
+  val noAllowRetryCnt = RegInit(0.U(64.W))
+
+  when (io.in.rx.rsp.fire && io.in.rx.rsp.bits.opcode === RetryAck) {
+    retryAckCnt := retryAckCnt + 1.U
+  }
+  when (io.in.rx.rsp.fire && io.in.rx.rsp.bits.opcode === PCrdGrant) {
+    pCrdGrantCnt := pCrdGrantCnt + 1.U
+  }
+  when (io.in.tx.req.fire && !io.in.tx.req.bits.allowRetry) {
+    noAllowRetryCnt := noAllowRetryCnt + 1.U
+  }
+
   dontTouch(io.out)
+  dontTouch(retryAckCnt)
+  dontTouch(pCrdGrantCnt)
+  dontTouch(noAllowRetryCnt)
 
   def setSrcID[T <: Bundle](in: DecoupledIO[T], srcID: UInt = 0.U): DecoupledIO[T] = {
     val out = Wire(in.cloneType)