[nvidia] Skip SAI discovery on ports on fast-boot

Given that modern systems have lots of ports, performing SAI discovery takes very long time, e.g. 8 sec for 256 port system. This has a big impact of fast-boot downtime and the discovery itself is not required for Nvidia platform fast-boot. Same applies to Nvidia fastfast-boot (aka warm-boot), yet needs to be tested separately. Signed-off-by: Stepan Blyschak <[email protected]>
sonic-net · Aug 21, 2024 · aa63de0 · aa63de0
1 parent 45ff42c
commit aa63de0
Show file tree

Hide file tree

Showing 9 changed files with 66 additions and 33 deletions.
diff --git a/configure.ac b/configure.ac
@@ -17,6 +17,7 @@ AX_ADD_AM_MACRO_STATIC([])
 
 AM_CONDITIONAL(SONIC_ASIC_PLATFORM_BAREFOOT,   test x$CONFIGURED_PLATFORM = xbarefoot)
 AM_CONDITIONAL(SONIC_ASIC_PLATFORM_BROADCOM,   test x$CONFIGURED_PLATFORM = xbroadcom)
+AM_CONDITIONAL(SONIC_ASIC_PLATFORM_MELLANOX,   test x$CONFIGURED_PLATFORM = xmellanox)
 
 AC_ARG_ENABLE(debug,
 [  --enable-debug          turn on debugging],

diff --git a/saiasiccmp/SaiSwitchAsic.cpp b/saiasiccmp/SaiSwitchAsic.cpp
@@ -272,7 +272,8 @@ std::set<sai_object_id_t> SaiSwitchAsic::getWarmBootDiscoveredVids() const
 
 void SaiSwitchAsic::onPostPortCreate(
         _In_ sai_object_id_t port_rid,
-        _In_ sai_object_id_t port_vid)
+        _In_ sai_object_id_t port_vid,
+        _In_ bool discoverPortObjects)
 {
     SWSS_LOG_ENTER();
 

diff --git a/saiasiccmp/SaiSwitchAsic.h b/saiasiccmp/SaiSwitchAsic.h
@@ -66,7 +66,8 @@ namespace saiasiccmp
 
             virtual void onPostPortCreate(
                     _In_ sai_object_id_t port_rid,
-                    _In_ sai_object_id_t port_vid) override;
+                    _In_ sai_object_id_t port_vid,
+                    _In_ bool discoverPortObjects = true) override;
 
             virtual void postPortRemove(
                     _In_ sai_object_id_t portRid) override;

diff --git a/syncd/Makefile.am b/syncd/Makefile.am
@@ -82,6 +82,10 @@ if SONIC_ASIC_PLATFORM_BROADCOM
 libSyncd_a_CXXFLAGS += -DMDIO_ACCESS_USE_NPU
 endif
 
+if SONIC_ASIC_PLATFORM_MELLANOX
+libSyncd_a_CPPFLAGS += -DSKIP_SAI_PORT_DISCOVERY_ON_FAST_BOOT
+endif
+
 libSyncdRequestShutdown_a_SOURCES = \
 									RequestShutdown.cpp \
 									RequestShutdownCommandLineOptions.cpp \

diff --git a/syncd/SaiSwitch.cpp b/syncd/SaiSwitch.cpp
@@ -948,46 +948,50 @@ void SaiSwitch::redisUpdatePortLaneMap(
 
 void SaiSwitch::onPostPortCreate(
         _In_ sai_object_id_t port_rid,
-        _In_ sai_object_id_t port_vid)
+        _In_ sai_object_id_t port_vid,
+        _In_ bool discoverPortObjects)
 {
     SWSS_LOG_ENTER();
 
-    SaiDiscovery sd(m_vendorSai);
+    if (discoverPortObjects)
+    {
+        SaiDiscovery sd(m_vendorSai);
 
-    auto discovered = sd.discover(port_rid);
+        auto discovered = sd.discover(port_rid);
 
-    auto defaultOidMap = sd.getDefaultOidMap();
+        auto defaultOidMap = sd.getDefaultOidMap();
 
-    // we need to merge default oid maps
+        // we need to merge default oid maps
 
-    for (auto& kvp: defaultOidMap)
-    {
-        for (auto& it: kvp.second)
+        for (auto& kvp: defaultOidMap)
         {
-            m_defaultOidMap[kvp.first][it.first] = it.second;
+            for (auto& it: kvp.second)
+            {
+                m_defaultOidMap[kvp.first][it.first] = it.second;
+            }
         }
-    }
 
-    SWSS_LOG_NOTICE("discovered %zu new objects (including port) after creating port VID: %s",
-            discovered.size(),
-            sai_serialize_object_id(port_vid).c_str());
+        SWSS_LOG_NOTICE("discovered %zu new objects (including port) after creating port VID: %s",
+                discovered.size(),
+                sai_serialize_object_id(port_vid).c_str());
 
-    m_discovered_rids.insert(discovered.begin(), discovered.end());
+        m_discovered_rids.insert(discovered.begin(), discovered.end());
 
-    SWSS_LOG_NOTICE("putting ALL new discovered objects to redis for port %s",
-            sai_serialize_object_id(port_vid).c_str());
+        SWSS_LOG_NOTICE("putting ALL new discovered objects to redis for port %s",
+                sai_serialize_object_id(port_vid).c_str());
 
-    for (sai_object_id_t rid: discovered)
-    {
-        /*
-         * We also could thing of optimizing this since it's one call to redis
-         * per rid, and probably this should be ATOMIC.
-         *
-         * NOTE: We are also storing read only object's here, like default
-         * virtual router, CPU, default trap group, etc.
-         */
+        for (sai_object_id_t rid: discovered)
+        {
+            /*
+            * We also could thing of optimizing this since it's one call to redis
+            * per rid, and probably this should be ATOMIC.
+            *
+            * NOTE: We are also storing read only object's here, like default
+            * virtual router, CPU, default trap group, etc.
+            */
 
-        redisSetDummyAsicStateForRealObjectId(rid);
+            redisSetDummyAsicStateForRealObjectId(rid);
+        }
     }
 
     redisUpdatePortLaneMap(port_rid);

diff --git a/syncd/SaiSwitch.h b/syncd/SaiSwitch.h
@@ -184,11 +184,12 @@ namespace syncd
              *
              * Performs actions needed after port creation. Will discover new
              * queues, ipgs and scheduler groups that belong to new created port,
-             * and updated ASIC DB accordingly.
+             * and updated ASIC DB accordingly when discoverPortObjects is true.
              */
             virtual void onPostPortCreate(
                     _In_ sai_object_id_t port_rid,
-                    _In_ sai_object_id_t port_vid) override;
+                    _In_ sai_object_id_t port_vid,
+                    _In_ bool discoverPortObjects = true) override;
 
             /**
              * @brief Post port remove.

diff --git a/syncd/SaiSwitchInterface.h b/syncd/SaiSwitchInterface.h
@@ -89,7 +89,8 @@ namespace syncd
 
             virtual void onPostPortCreate(
                     _In_ sai_object_id_t port_rid,
-                    _In_ sai_object_id_t port_vid) = 0;
+                    _In_ sai_object_id_t port_vid,
+                    _In_ bool discoverPortObjects = true) = 0;
 
             virtual void postPortRemove(
                     _In_ sai_object_id_t portRid) = 0;

diff --git a/syncd/Syncd.cpp b/syncd/Syncd.cpp
@@ -2001,7 +2001,7 @@ sai_status_t Syncd::processBulkOidCreate(
 
             if (objectType == SAI_OBJECT_TYPE_PORT)
             {
-                m_switches.at(switchVid)->onPostPortCreate(objectRids[idx], objectVids[idx]);
+                m_switches.at(switchVid)->onPostPortCreate(objectRids[idx], objectVids[idx], shouldDiscoverPortObjects());
             }
         }
     }
@@ -3130,7 +3130,7 @@ sai_status_t Syncd::processOidCreate(
 
         if (objectType == SAI_OBJECT_TYPE_PORT)
         {
-            m_switches.at(switchVid)->onPostPortCreate(objectRid, objectVid);
+            m_switches.at(switchVid)->onPostPortCreate(objectRid, objectVid, shouldDiscoverPortObjects());
         }
     }
 
@@ -5296,3 +5296,21 @@ syncd_restart_type_t Syncd::handleRestartQuery(
 
     return RequestShutdownCommandLineOptions::stringToRestartType(op);
 }
+
+bool Syncd::shouldDiscoverPortObjects() const
+{
+#ifdef SKIP_SAI_PORT_DISCOVERY_ON_FAST_BOOT
+    const bool discoverPortObjectsInFastBoot = false;
+#else
+    const bool discoverPortObjectsInFastBoot = true;
+#endif
+
+    // Comparing with m_veryFirstRun, so that we only skip discovery when switch is fast booting
+    // and not after it finished fast boot (e.g. port breakout after fast-reboot).
+    if ((m_commandLineOptions->m_startType == SAI_START_TYPE_FAST_BOOT) && m_veryFirstRun)
+    {
+        return discoverPortObjectsInFastBoot;
+    }
+
+    return true;
+}
diff --git a/syncd/Syncd.h b/syncd/Syncd.h
@@ -334,6 +334,8 @@ namespace syncd
             syncd_restart_type_t handleRestartQuery(
                     _In_ swss::NotificationConsumer &restartQuery);
 
+            bool shouldDiscoverPortObjects() const;
+
         private:
 
             /**