AXI Memory Over TL Serial Link

generators/chipyard/src/main/scala/ChipTop.scala

@@ -28,7 +28,7 @@ class ChipTop(implicit p: Parameters) extends LazyModule with BindingScope
   // The system module specified by BuildSystem
   lazy val lazySystem = LazyModule(p(BuildSystem)(p)).suggestName("system")
 
-  // The implicitClockSinkNode provides the implicit clock and reset for the System
+  // The implicitClockSinkNode provides the implicit clock and reset for the system (connected by clocking scheme)
   val implicitClockSinkNode = ClockSinkNode(Seq(ClockSinkParameters(name = Some("implicit_clock"))))
 
   // Generate Clocks and Reset

generators/chipyard/src/main/scala/Clocks.scala

@@ -47,6 +47,12 @@ case object ClockingSchemeKey extends Field[ChipTop => Unit](ClockingSchemeGener
   */
 case object ClockFrequencyAssignersKey extends Field[Seq[(String) => Option[Double]]](Seq.empty)
 case object DefaultClockFrequencyKey extends Field[Double]()
+case object ReferenceClockTrackerKey extends Field[ReferenceClockTracker](new ReferenceClockTracker)
+class ReferenceClockTracker {
+  private var _refFreqMHz: Option[Double] = None
+  def set(freqMHz: Double): Unit = { _refFreqMHz = Some(freqMHz) }
+  def get: Option[Double] = { _refFreqMHz }
+}
 
 class ClockNameMatchesAssignment(name: String, fMHz: Double) extends Config((site, here, up) => {
   case ClockFrequencyAssignersKey => up(ClockFrequencyAssignersKey, site) ++
@@ -75,28 +81,33 @@ object ClockingSchemeGenerators {
     }
 
     val aggregator = LazyModule(new ClockGroupAggregator("allClocks")).node
+    // provides the implicit clock to the system
     (chiptop.implicitClockSinkNode
       := ClockGroup()
       := aggregator)
+    // provides the system clock (ex. the bus clocks)
     (systemAsyncClockGroup
       :*= resetSetter
       :*= ClockGroupNamePrefixer()
       :*= aggregator)
 
     val referenceClockSource =  ClockSourceNode(Seq(ClockSourceParameters()))
+    val dividerOnlyClkGenerator = DividerOnlyClockGenerator()
+    // provides all the divided clocks (from the top-level clock)
     (aggregator
       := ClockGroupFrequencySpecifier(p(ClockFrequencyAssignersKey), p(DefaultClockFrequencyKey))
       := ClockGroupResetSynchronizer()
-      := DividerOnlyClockGenerator()
+      := dividerOnlyClkGenerator.node
       := referenceClockSource)
 
-
     InModuleBody {
       val clock_wire = Wire(Input(Clock()))
       val reset_wire = GenerateReset(chiptop, clock_wire)
       val (clock_io, clockIOCell) = IOCell.generateIOFromSignal(clock_wire, "clock")
       chiptop.iocells ++= clockIOCell
 
+      p(ReferenceClockTrackerKey).set(dividerOnlyClkGenerator.module.referenceFreq)
+
       referenceClockSource.out.unzip._1.map { o =>
         o.clock := clock_wire
         o.reset := reset_wire

generators/chipyard/src/main/scala/ConfigFragments.scala

@@ -193,7 +193,20 @@ class WithTLBackingMemory extends Config((site, here, up) => {
   case ExtTLMem => up(ExtMem, site) // enable TL backing memory
 })
 
-class WithTileFrequency(fMHz: Double) extends ClockNameContainsAssignment("core", fMHz)
+class WithSerialTLBackingMemory extends Config((site, here, up) => {
+  case ExtMem => None
+  case SerialTLKey => up(SerialTLKey, site).map { k => k.copy(
+    memParams = {
+      val memPortParams = up(ExtMem, site).get
+      require(memPortParams.nMemoryChannels == 1)
+      memPortParams.master
+    },
+    isMemoryDevice = true
+  )}
+})
+
+class WithTileFrequency(fMHz: Double) extends ClockNameContainsAssignment("tile", fMHz)
+class WithSpecificTileFrequency(hartId: Int, fMHz: Double) extends chipyard.ClockNameContainsAssignment(s"tile_$hartId", fMHz)
 
 class WithPeripheryBusFrequencyAsDefault extends Config((site, here, up) => {
   case DefaultClockFrequencyKey => (site(PeripheryBusKey).dtsFrequency.get / (1000 * 1000)).toDouble

generators/chipyard/src/main/scala/HarnessBinders.scala

@@ -21,7 +21,7 @@ import barstools.iocell.chisel._
 
 import testchipip._
 
-import chipyard.HasHarnessSignalReferences
+import chipyard.{HasHarnessSignalReferences, HarnessClockInstantiatorKey}
 import chipyard.iobinders.GetSystemParameters
 
 import tracegen.{TraceGenSystemModuleImp}
@@ -83,7 +83,9 @@ class WithGPIOTiedOff extends OverrideHarnessBinder({
 // DOC include start: WithUARTAdapter
 class WithUARTAdapter extends OverrideHarnessBinder({
   (system: HasPeripheryUARTModuleImp, th: HasHarnessSignalReferences, ports: Seq[UARTPortIO]) => {
-    UARTAdapter.connect(ports)(system.p)
+    withClockAndReset(th.harnessClock, th.harnessReset) {
+      UARTAdapter.connect(ports)(system.p)
+    }
   }
 })
 // DOC include end: WithUARTAdapter
@@ -139,6 +141,46 @@ class WithSimAXIMem extends OverrideHarnessBinder({
   }
 })
 
+class WithSimAXIMemOverSerialTL extends OverrideHarnessBinder({
+  (system: CanHavePeripheryTLSerial, th: HasHarnessSignalReferences, ports: Seq[ClockedIO[SerialIO]]) => {
+    implicit val p = chipyard.iobinders.GetSystemParameters(system)
+
+    p(SerialTLKey).map({ sVal =>
+      // currently only the harness AXI port supports a passthrough clock
+      require(sVal.axiMemOverSerialTLParams.isDefined)
+      val axiDomainParams = sVal.axiMemOverSerialTLParams.get
+
+      val memFreq: Double = axiDomainParams.axiClockParams match {
+        case Some(clkParams) => clkParams.clockFreqMHz * 1000000
+        case None => {
+          // get freq. from what the master of the serial link specifies
+          system.asInstanceOf[HasTileLinkLocations].locateTLBusWrapper(p(SerialTLAttachKey).masterWhere).dtsFrequency.get.toDouble
+        }
+      }
+
+      ports.map({ port =>
+        val harnessMultiClockAXIRAM = SerialAdapter.connectHarnessMultiClockAXIRAM(
+          system.serdesser.get,
+          port,
+          p(HarnessClockInstantiatorKey).getClockBundleWire("mem_over_serial_tl_clock", memFreq),
+          th.harnessReset)
+        val success = SerialAdapter.connectSimSerial(harnessMultiClockAXIRAM.module.io.tsi_ser, port.clock, th.harnessReset.asBool)
+        when (success) { th.success := true.B }
+
+        // connect SimDRAM from the AXI port coming from the harness multi clock axi ram
+        (harnessMultiClockAXIRAM.mem_axi4 zip harnessMultiClockAXIRAM.memNode.edges.in).map { case (axi_port, edge) =>
+          val memSize = sVal.memParams.size
+          val lineSize = p(CacheBlockBytes)
+          val mem = Module(new SimDRAM(memSize, lineSize, BigInt(memFreq.toInt), edge.bundle)).suggestName("simdram")
+          mem.io.axi <> axi_port.bits
+          mem.io.clock := axi_port.clock
+          mem.io.reset := axi_port.reset
+        }
+      })
+    })
+  }
+})
+
 class WithBlackBoxSimMem(additionalLatency: Int = 0) extends OverrideHarnessBinder({
   (system: CanHaveMasterAXI4MemPort, th: HasHarnessSignalReferences, ports: Seq[ClockedAndResetIO[AXI4Bundle]]) => {
     val p: Parameters = chipyard.iobinders.GetSystemParameters(system)

generators/chipyard/src/main/scala/IOBinders.scala

@@ -260,7 +260,6 @@ class WithSerialTLIOCells extends OverrideIOBinder({
   }).getOrElse((Nil, Nil))
 })
 
-
 class WithAXI4MemPunchthrough extends OverrideLazyIOBinder({
   (system: CanHaveMasterAXI4MemPort) => {
     implicit val p: Parameters = GetSystemParameters(system)

generators/chipyard/src/main/scala/TestHarness.scala

@@ -1,12 +1,16 @@
 package chipyard
 
 import chisel3._
-import scala.collection.mutable.{ArrayBuffer}
+
+import scala.collection.mutable.{ArrayBuffer, HashMap}
 import freechips.rocketchip.diplomacy.{LazyModule}
 import freechips.rocketchip.config.{Field, Parameters}
+import freechips.rocketchip.util.{ResetCatchAndSync}
+import freechips.rocketchip.prci.{ClockBundle, ClockBundleParameters, ClockSinkParameters, ClockParameters}
 
 import chipyard.harness.{ApplyHarnessBinders, HarnessBinders}
 import chipyard.iobinders.HasIOBinders
+import chipyard.clocking.{SimplePllConfiguration, ClockDividerN}
 
 // -------------------------------
 // Chipyard Test Harness
@@ -25,6 +29,47 @@ trait HasHarnessSignalReferences {
   def success: Bool
 }
 
+class HarnessClockInstantiator {
+  private var _clockMap: HashMap[String, (Double, ClockBundle)] = HashMap.empty
+
+  // request a clock bundle at a particular frequency
+  def getClockBundleWire(name: String, freqRequested: Double): ClockBundle = {
+    val clockBundle = Wire(new ClockBundle(ClockBundleParameters()))
+    _clockMap(name) = (freqRequested, clockBundle)
+    clockBundle
+  }
+
+  // connect all clock wires specified to a divider only PLL
+  def instantiateHarnessDividerPLL(refClock: ClockBundle): Unit = {
+    val sinks = _clockMap.map({ case (name, (freq, bundle)) =>
+      ClockSinkParameters(take=Some(ClockParameters(freqMHz=freq/1000000)),name=Some(name))
+    }).toSeq
+
+    val pllConfig = new SimplePllConfiguration("harnessDividerOnlyClockGenerator", sinks)
+    pllConfig.emitSummaries()
+
+    val dividedClocks = HashMap[Int, Clock]()
+    def instantiateDivider(div: Int): Clock = {
+      val divider = Module(new ClockDividerN(div))
+      divider.suggestName(s"ClockDivideBy${div}")
+      divider.io.clk_in := refClock.clock
+      dividedClocks(div) = divider.io.clk_out
+      divider.io.clk_out
+    }
+
+    // TODO: on the implicit clock just create a passthrough (don't instantiate a divider + reset catch)
+    // connect wires to clock source
+    for (sinkParams <- sinks) {
+      val div = pllConfig.sinkDividerMap(sinkParams)
+      val divClock = dividedClocks.getOrElse(div, instantiateDivider(div))
+      _clockMap(sinkParams.name.get)._2.clock := divClock
+      _clockMap(sinkParams.name.get)._2.reset := ResetCatchAndSync(divClock, refClock.reset.asBool)
+    }
+  }
+}
+
+case object HarnessClockInstantiatorKey extends Field[HarnessClockInstantiator](new HarnessClockInstantiator)
+
 class TestHarness(implicit val p: Parameters) extends Module with HasHarnessSignalReferences {
   val io = IO(new Bundle {
     val success = Output(Bool())
@@ -34,17 +79,23 @@ class TestHarness(implicit val p: Parameters) extends Module with HasHarnessSign
   val dut = Module(lazyDut.module)
   io.success := false.B
 
-  val harnessClock = clock
-  val harnessReset = WireInit(reset)
+  val (harnessClock, harnessReset, dutReset) = {
+    val freqMHz = p(ReferenceClockTrackerKey).get.getOrElse(100.0) // default to 100MHz
+    val bundle = p(HarnessClockInstantiatorKey).getClockBundleWire("implicit_harness_clock", freqMHz*1000000.0)
+    (bundle.clock, WireInit(bundle.reset), bundle.reset.asAsyncReset)
+  }
   val success = io.success
 
-  val dutReset = reset.asAsyncReset
-
   lazyDut match { case d: HasTestHarnessFunctions =>
     d.harnessFunctions.foreach(_(this))
   }
   lazyDut match { case d: HasIOBinders =>
     ApplyHarnessBinders(this, d.lazySystem, d.portMap)
   }
+
+  val implicitHarnessClockBundle = Wire(new ClockBundle(ClockBundleParameters()))
+  implicitHarnessClockBundle.clock := clock
+  implicitHarnessClockBundle.reset := reset
+  p(HarnessClockInstantiatorKey).instantiateHarnessDividerPLL(implicitHarnessClockBundle)
 }
 

generators/chipyard/src/main/scala/clocking/DividerOnlyClockGenerator.scala

@@ -146,5 +146,5 @@ class DividerOnlyClockGenerator(pllName: String)(implicit p: Parameters, valName
 }
 
 object DividerOnlyClockGenerator {
-  def apply()(implicit p: Parameters, valName: ValName) = LazyModule(new DividerOnlyClockGenerator(valName.name)).node
+  def apply()(implicit p: Parameters, valName: ValName) = LazyModule(new DividerOnlyClockGenerator(valName.name))
 }

generators/chipyard/src/main/scala/config/AbstractConfig.scala

@@ -54,4 +54,3 @@ class AbstractConfig extends Config(
   new freechips.rocketchip.subsystem.WithNExtTopInterrupts(0) ++ // no external interrupts
   new chipyard.WithMulticlockCoherentBusTopology ++              // hierarchical buses including mbus+l2
   new freechips.rocketchip.system.BaseConfig)                    // "base" rocketchip system
-

generators/chipyard/src/main/scala/config/RocketConfigs.scala

@@ -1,7 +1,8 @@
 package chipyard
 
 import freechips.rocketchip.config.{Config}
-import freechips.rocketchip.diplomacy.{AsynchronousCrossing}
+import freechips.rocketchip.diplomacy.{AsynchronousCrossing, RationalCrossing}
+import freechips.rocketchip.util.{SlowToFast}
 
 // --------------
 // Rocket Configs
@@ -212,3 +213,22 @@ class LBWIFRocketConfig extends Config(
   new freechips.rocketchip.subsystem.WithNoMemPort ++       // remove AXI4 backing memory
   new freechips.rocketchip.subsystem.WithNBigCores(1) ++
   new chipyard.config.AbstractConfig)
+
+class MulticlockAXIOverSerialConfig extends Config(
+  new chipyard.config.WithFbusToSbusCrossingType(RationalCrossing(SlowToFast)) ++
+
+  new chipyard.config.WithSystemBusFrequencyAsDefault ++
+  new chipyard.config.WithSystemBusFrequency(4000) ++
+  new chipyard.config.WithPeripheryBusFrequency(4000) ++
+  new chipyard.config.WithMemoryBusFrequency(4000) ++
+
+  new chipyard.config.WithFrontBusFrequency(4000 / 2) ++
+
+  new chipyard.config.WithFbusToSbusCrossingType(RationalCrossing(SlowToFast)) ++
+  new testchipip.WithAsynchronousSerialSlaveCrossing ++
+
+  new chipyard.harness.WithSimAXIMemOverSerialTL ++ // add SimDRAM DRAM model for axi4 backing memory over the SerDes link, if axi4 mem is enabled
+  new chipyard.config.WithSerialTLBackingMemory ++ // remove axi4 mem port in favor of SerialTL memory
+
+  new freechips.rocketchip.subsystem.WithNBigCores(1) ++
+  new chipyard.config.AbstractConfig)

generators/firechip/src/main/scala/BridgeBinders.scala

@@ -98,7 +98,40 @@ class WithUARTBridge extends OverrideHarnessBinder({
 class WithBlockDeviceBridge extends OverrideHarnessBinder({
   (system: CanHavePeripheryBlockDevice, th: FireSim, ports: Seq[ClockedIO[BlockDeviceIO]]) => {
     implicit val p: Parameters = GetSystemParameters(system)
-    ports.map { b => BlockDevBridge(b.clock, b.bits, th.harnessReset.toBool) }
+    ports.map { b => BlockDevBridge(b.clock, b.bits, th.harnessReset.asBool) }
+    Nil
+  }
+})
+
+class WithAXIOverSerialTLCombinedBridges extends OverrideHarnessBinder({
+  (system: CanHavePeripheryTLSerial, th: FireSim, ports: Seq[SerialAndPassthroughClockResetIO]) => {
+    implicit val p = GetSystemParameters(system)
+
+    p(SerialTLKey).map({ sVal =>
+      // require having memory over the serdes link
+      require(sVal.isMemoryDevice)
+
+      ports.map({ port =>
+        val offchipNetwork = SerialAdapter.connectHarnessMultiClockAXIRAM(system.serdesser.get, port, th.harnessReset)
+        SerialBridge(port.clocked_serial.clock, offchipNetwork.module.io.tsi_ser, Some(MainMemoryConsts.globalName))
+
+        // connect SimAxiMem
+        (offchipNetwork.mem_axi4 zip offchipNetwork.memAXI4Node.edges.in).map { case (axi4, edge) =>
+          val nastiKey = NastiParameters(axi4.r.bits.data.getWidth,
+                                        axi4.ar.bits.addr.getWidth,
+                                        axi4.ar.bits.id.getWidth)
+          system match {
+            case s: BaseSubsystem => FASEDBridge(port.passthrough_clock_reset.clock, axi4, port.passthrough_clock_reset.reset.asBool,
+              CompleteConfig(p(firesim.configs.MemModelKey),
+                            nastiKey,
+                            Some(AXI4EdgeSummary(edge)),
+                            Some(MainMemoryConsts.globalName)))
+            case _ => throw new Exception("Attempting to attach FASED Bridge to misconfigured design")
+          }
+        }
+      })
+    })
+
     Nil
   }
 })