AXI Memory Over TL Serial Link

generators/chipyard/src/main/scala/Clocks.scala

@@ -75,15 +75,18 @@ object ClockingSchemeGenerators {
     }
 
     val aggregator = LazyModule(new ClockGroupAggregator("allClocks")).node
+    // provides the implicit clock to the system
     (chiptop.implicitClockSinkNode
       := ClockGroup()
       := aggregator)
+    // provides the system clock (ex. the bus clocks)
     (systemAsyncClockGroup
       :*= resetSetter
       :*= ClockGroupNamePrefixer()
       :*= aggregator)
 
     val referenceClockSource =  ClockSourceNode(Seq(ClockSourceParameters()))
+    // provides all the divided clocks (from the top-level clock)
     (aggregator
       := ClockGroupFrequencySpecifier(p(ClockFrequencyAssignersKey), p(DefaultClockFrequencyKey))
       := ClockGroupResetSynchronizer()

generators/chipyard/src/main/scala/ConfigFragments.scala

@@ -193,7 +193,20 @@ class WithTLBackingMemory extends Config((site, here, up) => {
   case ExtTLMem => up(ExtMem, site) // enable TL backing memory
 })
 
-class WithTileFrequency(fMHz: Double) extends ClockNameContainsAssignment("core", fMHz)
+class WithSerialTLBackingMemory extends Config((site, here, up) => {
+  case ExtMem => None
+  case SerialTLKey => up(SerialTLKey, site).map { k => k.copy(
+    memParams = {
+      val memPortParams = up(ExtMem, site).get
+      require(memPortParams.nMemoryChannels == 1)
+      memPortParams.master
+    },
+    isMemoryDevice = true
+  )}
+})
+
+class WithTileFrequency(fMHz: Double) extends ClockNameContainsAssignment("tile", fMHz)
+class WithSpecificTileFrequency(hartId: Int, fMHz: Double) extends chipyard.ClockNameContainsAssignment(s"tile_$hartId", fMHz)
 
 class WithPeripheryBusFrequencyAsDefault extends Config((site, here, up) => {
   case DefaultClockFrequencyKey => (site(PeripheryBusKey).dtsFrequency.get / (1000 * 1000)).toDouble

generators/chipyard/src/main/scala/HarnessBinders.scala

@@ -139,6 +139,44 @@ class WithSimAXIMem extends OverrideHarnessBinder({
   }
 })
 
+class WithSimAXIMemOverSerialTL extends OverrideHarnessBinder({
+  (system: CanHavePeripheryTLSerial, th: HasHarnessSignalReferences, ports: Seq[SerialAndPassthroughClockResetIO]) => {
+    implicit val p = chipyard.iobinders.GetSystemParameters(system)
+
+    p(SerialTLKey).map({ sVal =>
+      // currently only the harness AXI port supports a passthrough clock
+      require(sVal.axiMemOverSerialTLParams.isDefined)
+      val axiDomainParams = sVal.axiMemOverSerialTLParams.get
+
+      val memFreq = axiDomainParams.axiClockParams match {
+        case Some(clkParams) => {
+          BigInt(clkParams.clockFreqMHz.toInt)*1000000
+        }
+        case None => {
+          // get freq. from what the master bus specifies
+          system.asInstanceOf[HasTileLinkLocations].locateTLBusWrapper(p(SerialTLAttachKey).masterWhere).dtsFrequency.get
+        }
+      }
+
+      ports.map({ port =>
+        val harnessMultiClockAXIRAM = SerialAdapter.connectHarnessMultiClockAXIRAM(system.serdesser.get, port, th.harnessReset)
+        val success = SerialAdapter.connectSimSerial(harnessMultiClockAXIRAM.module.io.tsi_ser, port.clocked_serial.clock, th.harnessReset.asBool)
+        when (success) { th.success := true.B }
+
+        // connect SimDRAM from the AXI port coming from the harness multi clock axi ram
+        (harnessMultiClockAXIRAM.mem_axi4 zip harnessMultiClockAXIRAM.memNode.edges.in).map { case (axi_port, edge) =>
+          val memSize = sVal.memParams.size
+          val lineSize = p(CacheBlockBytes)
+          val mem = Module(new SimDRAM(memSize, lineSize, memFreq, edge.bundle)).suggestName("simdram")
+          mem.io.axi <> axi_port.bits
+          mem.io.clock := axi_port.clock
+          mem.io.reset := axi_port.reset
+        }
+      })
+    })
+  }
+})
+
 class WithBlackBoxSimMem(additionalLatency: Int = 0) extends OverrideHarnessBinder({
   (system: CanHaveMasterAXI4MemPort, th: HasHarnessSignalReferences, ports: Seq[ClockedAndResetIO[AXI4Bundle]]) => {
     val p: Parameters = chipyard.iobinders.GetSystemParameters(system)

generators/chipyard/src/main/scala/IOBinders.scala

@@ -11,7 +11,7 @@ import freechips.rocketchip.subsystem._
 import freechips.rocketchip.system.{SimAXIMem}
 import freechips.rocketchip.amba.axi4.{AXI4Bundle, AXI4SlaveNode, AXI4MasterNode, AXI4EdgeParameters}
 import freechips.rocketchip.util._
-import freechips.rocketchip.prci.{ClockSinkNode, ClockSinkParameters}
+import freechips.rocketchip.prci.{ClockSinkNode, ClockSinkParameters, ClockParameters, ClockGroup, ClockBundle, ClockBundleParameters}
 import freechips.rocketchip.groundtest.{GroundTestSubsystemModuleImp, GroundTestSubsystem}
 
 import sifive.blocks.devices.gpio._
@@ -260,6 +260,48 @@ class WithSerialTLIOCells extends OverrideIOBinder({
   }).getOrElse((Nil, Nil))
 })
 
+class WithSerialTLAndPassthroughClockPunchthrough extends OverrideLazyIOBinder({
+  (system: CanHavePeripheryTLSerial) => system.serial_tl.map({ serial_io =>
+    implicit val p: Parameters = GetSystemParameters(system)
+
+    val sys = system.asInstanceOf[BaseSubsystem]
+
+    require(p(SerialTLKey).isDefined)
+    val sVal = p(SerialTLKey).get
+
+    // currently only the harness AXI port supports a passthrough clock
+    require(sVal.axiMemOverSerialTLParams.isDefined)
+    val axiDomainParams = sVal.axiMemOverSerialTLParams.get
+
+    val clockSinkNode = axiDomainParams.axiClockParams.map({ clkParams =>
+      // request clock to pass along
+      val node = ClockSinkNode(Seq(ClockSinkParameters(take = Some(ClockParameters(freqMHz = clkParams.clockFreqMHz)))))
+      (node
+        := ClockGroup()(p, ValName("mem_over_serialtl_domain"))
+        := sys.asyncClockGroupsNode)
+      node
+    })
+
+    def clockBundle = clockSinkNode match {
+      case Some(node) => node.in.head._1
+      case None => {
+        val dontCareClockBundle = new ClockBundle(ClockBundleParameters())
+        dontCareClockBundle.clock := DontCare
+        dontCareClockBundle.reset := DontCare
+        dontCareClockBundle
+      }
+    }
+
+    InModuleBody {
+      val port = IO(new SerialAndPassthroughClockResetIO(sVal.width)).suggestName(s"serial_tl_passthrough_clk")
+      port.clocked_serial <> serial_io
+      port.passthrough_clock_reset <> clockBundle
+
+      // return the ports and no IO cells
+      (Seq(port), Nil)
+    }
+  }).getOrElse(InModuleBody{(Nil, Nil)}).asInstanceOf[ModuleValue[IOBinderTuple]]
+})
 
 class WithAXI4MemPunchthrough extends OverrideLazyIOBinder({
   (system: CanHaveMasterAXI4MemPort) => {

generators/chipyard/src/main/scala/config/AbstractConfig.scala

@@ -54,4 +54,3 @@ class AbstractConfig extends Config(
   new freechips.rocketchip.subsystem.WithNExtTopInterrupts(0) ++ // no external interrupts
   new chipyard.WithMulticlockCoherentBusTopology ++              // hierarchical buses including mbus+l2
   new freechips.rocketchip.system.BaseConfig)                    // "base" rocketchip system
-

generators/chipyard/src/main/scala/config/RocketConfigs.scala

@@ -1,7 +1,8 @@
 package chipyard
 
 import freechips.rocketchip.config.{Config}
-import freechips.rocketchip.diplomacy.{AsynchronousCrossing}
+import freechips.rocketchip.diplomacy.{AsynchronousCrossing, RationalCrossing}
+import freechips.rocketchip.util.{SlowToFast}
 
 // --------------
 // Rocket Configs
@@ -212,3 +213,26 @@ class LBWIFRocketConfig extends Config(
   new freechips.rocketchip.subsystem.WithNoMemPort ++       // remove AXI4 backing memory
   new freechips.rocketchip.subsystem.WithNBigCores(1) ++
   new chipyard.config.AbstractConfig)
+
+class MulticlockAXIOverSerialConfig extends Config(
+  new chipyard.config.WithFbusToSbusCrossingType(RationalCrossing(SlowToFast)) ++
+
+  new chipyard.config.WithSystemBusFrequencyAsDefault ++
+  new chipyard.config.WithSystemBusFrequency(4000) ++
+  new chipyard.config.WithPeripheryBusFrequency(4000) ++
+  new chipyard.config.WithMemoryBusFrequency(4000) ++
+
+  new chipyard.config.WithFrontBusFrequency(4000 / 2) ++
+
+  new chipyard.config.WithFbusToSbusCrossingType(RationalCrossing(SlowToFast)) ++
+  new testchipip.WithAsynchronousSerialSlaveCrossing ++
+
+  new chipyard.harness.WithSimAXIMemOverSerialTL ++ // add SimDRAM DRAM model for axi4 backing memory over the SerDes link, if axi4 mem is enabled
+  new chipyard.iobinders.WithSerialTLAndPassthroughClockPunchthrough ++ // add new clock for axi domain over serdes and passthrough ios
+  new chipyard.config.WithSerialTLBackingMemory ++ // remove axi4 mem port in favor of SerialTL memory
+  new testchipip.WithBlockDeviceLocations(
+    freechips.rocketchip.subsystem.PBUS,
+    freechips.rocketchip.subsystem.PBUS) ++ // put block device fully on PBUS to avoid clock crossings
+
+  new freechips.rocketchip.subsystem.WithNBigCores(1) ++
+  new chipyard.config.AbstractConfig)

generators/firechip/src/main/scala/BridgeBinders.scala

@@ -98,7 +98,40 @@ class WithUARTBridge extends OverrideHarnessBinder({
 class WithBlockDeviceBridge extends OverrideHarnessBinder({
   (system: CanHavePeripheryBlockDevice, th: FireSim, ports: Seq[ClockedIO[BlockDeviceIO]]) => {
     implicit val p: Parameters = GetSystemParameters(system)
-    ports.map { b => BlockDevBridge(b.clock, b.bits, th.harnessReset.toBool) }
+    ports.map { b => BlockDevBridge(b.clock, b.bits, th.harnessReset.asBool) }
+    Nil
+  }
+})
+
+class WithAXIOverSerialTLCombinedBridges extends OverrideHarnessBinder({
+  (system: CanHavePeripheryTLSerial, th: FireSim, ports: Seq[SerialAndPassthroughClockResetIO]) => {
+    implicit val p = GetSystemParameters(system)
+
+    p(SerialTLKey).map({ sVal =>
+      // require having memory over the serdes link
+      require(sVal.isMemoryDevice)
+
+      ports.map({ port =>
+        val offchipNetwork = SerialAdapter.connectHarnessMultiClockAXIRAM(system.serdesser.get, port, th.harnessReset)
+        SerialBridge(port.clocked_serial.clock, offchipNetwork.module.io.tsi_ser, Some(MainMemoryConsts.globalName))
+
+        // connect SimAxiMem
+        (offchipNetwork.mem_axi4 zip offchipNetwork.memAXI4Node.edges.in).map { case (axi4, edge) =>
+          val nastiKey = NastiParameters(axi4.r.bits.data.getWidth,
+                                        axi4.ar.bits.addr.getWidth,
+                                        axi4.ar.bits.id.getWidth)
+          system match {
+            case s: BaseSubsystem => FASEDBridge(port.passthrough_clock_reset.clock, axi4, port.passthrough_clock_reset.reset.asBool,
+              CompleteConfig(p(firesim.configs.MemModelKey),
+                            nastiKey,
+                            Some(AXI4EdgeSummary(edge)),
+                            Some(MainMemoryConsts.globalName)))
+            case _ => throw new Exception("Attempting to attach FASED Bridge to misconfigured design")
+          }
+        }
+      })
+    })
+
     Nil
   }
 })

generators/firechip/src/main/scala/TargetConfigs.scala

@@ -59,26 +59,13 @@ class WithNIC extends icenet.WithIceNIC(inBufFlits = 8192, ctrlQueueDepth = 64)
 class WithNVDLALarge extends nvidia.blocks.dla.WithNVDLA("large")
 class WithNVDLASmall extends nvidia.blocks.dla.WithNVDLA("small")
 
-
-// Tweaks that are generally applied to all firesim configs
-class WithFireSimConfigTweaks extends Config(
+class WithFireSimConfigTweaksWithoutClocking extends Config(
   // Required: Bake in the default FASED memory model
   new WithDefaultMemModel ++
   // Required*: Uses FireSim ClockBridge and PeekPokeBridge to drive the system with a single clock/reset
   new WithFireSimSimpleClocks ++
   // Required*: When using FireSim-as-top to provide a correct path to the target bootrom source
   new WithBootROM ++
-  // Optional*: Removing this will require adjusting the UART baud rate and
-  // potential target-software changes to properly capture UART output
-  new chipyard.config.WithPeripheryBusFrequency(3200.0) ++
-  // Optional: These three configs put the DRAM memory system in it's own clock domian.
-  // Removing the first config will result in the FASED timing model running
-  // at the pbus freq (above, 3.2 GHz), which is outside the range of valid DDR3 speedgrades.
-  // 1 GHz matches the FASED default, using some other frequency will require
-  // runnings the FASED runtime configuration generator to generate faithful DDR3 timing values.
-  new chipyard.config.WithMemoryBusFrequency(1000.0) ++
-  new chipyard.config.WithAsynchrousMemoryBusCrossing ++
-  new testchipip.WithAsynchronousSerialSlaveCrossing ++
   // Required: Existing FAME-1 transform cannot handle black-box clock gates
   new WithoutClockGating ++
   // Required*: Removes thousands of assertions that would be synthesized (* pending PriorityMux bugfix)
@@ -87,10 +74,6 @@ class WithFireSimConfigTweaks extends Config(
   new chipyard.config.WithTraceIO ++
   // Optional: Request 16 GiB of target-DRAM by default (can safely request up to 32 GiB on F1)
   new freechips.rocketchip.subsystem.WithExtMemSize((1 << 30) * 16L) ++
-  // Required: Adds IO to attach SerialBridge. The SerialBridges is responsible
-  // for signalling simulation termination under simulation success. This fragment can
-  // be removed if you supply an auxiliary bridge that signals simulation termination
-  new testchipip.WithDefaultSerialTL ++
   // Optional: Removing this will require using an initramfs under linux
   new testchipip.WithBlockDevice ++
   // Required*: Scale default baud rate with periphery bus frequency
@@ -99,6 +82,23 @@ class WithFireSimConfigTweaks extends Config(
   new chipyard.config.WithNoDebug
 )
 
+// Tweaks that are generally applied to all firesim configs
+class WithFireSimConfigTweaks extends Config(
+  // Optional*: Removing this will require adjusting the UART baud rate and
+  // potential target-software changes to properly capture UART output
+  new chipyard.config.WithPeripheryBusFrequency(3200.0) ++
+  // Optional: These three configs put the DRAM memory system in it's own clock domian.
+  // Removing the first config will result in the FASED timing model running
+  // at the pbus freq (above, 3.2 GHz), which is outside the range of valid DDR3 speedgrades.
+  // 1 GHz matches the FASED default, using some other frequency will require
+  // runnings the FASED runtime configuration generator to generate faithful DDR3 timing values.
+  new chipyard.config.WithMemoryBusFrequency(1000.0) ++
+  new chipyard.config.WithAsynchrousMemoryBusCrossing ++
+  new testchipip.WithAsynchronousSerialSlaveCrossing ++
+  // Tweaks that are independent from multi-clock
+  new WithFireSimConfigTweaksWithoutClocking
+)
+
 /*******************************************************************************
 * Full TARGET_CONFIG configurations. These set parameters of the target being
 * simulated.
@@ -204,6 +204,14 @@ class FireSimMulticlockRocketConfig extends Config(
   new freechips.rocketchip.subsystem.WithRationalRocketTiles ++   // Add rational crossings between RocketTile and uncore
   new FireSimRocketConfig)
 
+class FireSimMulticlockAXIOverSerialConfig extends Config(
+  new WithAXIOverSerialTLCombinedBridges ++ // use combined bridge to connect to axi mem over serial
+  new WithDefaultFireSimBridges ++
+  new WithDefaultMemModel ++
+  new WithFireSimConfigTweaksWithoutClocking ++ // don't inherit firesim clocking
+  new chipyard.MulticlockAXIOverSerialConfig
+)
+
 //**********************************************************************************
 // System with 16 LargeBOOMs that can be simulated with Golden Gate optimizations
 // - Requires MTModels and MCRams mixins as prefixes to the platform config
@@ -215,3 +223,4 @@ class FireSim16LargeBoomConfig extends Config(
   new WithFireSimConfigTweaks ++
   new boom.common.WithNLargeBooms(16) ++
   new chipyard.config.AbstractConfig)
+