Ensure AutoGreylist correctly deals with newly whitelisted projects

This change introduces a map in the projects registry (whitelist)
that tracks when a project was first added. This is used in the
AutoGreylist::RefreshWithSuperblock method.

When projects are newly added to the whitelist, their first entry
(and resultant superblock) will be close to the head of the chain,
well within the 40 block lookback to execute the greylist ruleset.

The update loop in RefreshWithSuperblock will only post
greylist entries (remember it goes backwards from the present)
when the entry timestamp is greater than or equal to the first
project entry for that project (i.e. when it was first put on the
whitelist).

This is required to ensure the ZCD and WAS rules work correctly
while newly whitelisted projects do not have the full 40 SBs to
sample.

src/gridcoin/beacon.cpp

@@ -1239,14 +1239,16 @@ void BeaconRegistry::Deactivate(const uint256 superblock_hash)
 //! \param recnum
 //! \param key_type
 //!
-template<> void BeaconRegistry::BeaconDB::HandleCurrentHistoricalEntries(GRC::BeaconRegistry::BeaconMap& entries,
-                                               GRC::BeaconRegistry::PendingBeaconMap& pending_entries,
+ template<> void BeaconRegistry::BeaconDB::HandleCurrentHistoricalEntries(GRC::BeaconRegistry::BeaconMap& entries,
+                                                               GRC::BeaconRegistry::PendingBeaconMap& pending_entries,
                                                                std::set<Beacon_ptr>& expired_entries,
-                                               const Beacon& entry,
-                                               entry_ptr& historical_entry_ptr,
-                                               const uint64_t& recnum,
-                                               const std::string& key_type)
-{
+                                                               GRC::BeaconRegistry::BeaconMap& first_entries,
+                                                               const Beacon& entry,
+                                                               entry_ptr& historical_entry_ptr,
+                                                               const uint64_t& recnum,
+                                                               const std::string& key_type,
+                                                               const bool& populate_first_entries)
+ {
     // Note that in this specialization, entry.m_cpid and entry.GetId() are used for the map keys. In the general template,
     // entry.Key() is used (which here is the same as entry.m_cpid). No generalized method to implement entry.PendingKey()
     // has been implemented up to this point, because the pending map is actually only used here in the beacon
@@ -1366,7 +1368,7 @@ template<> void BeaconRegistry::BeaconDB::HandleCurrentHistoricalEntries(GRC::Be
 
 int BeaconRegistry::Initialize()
 {
-    int height = m_beacon_db.Initialize(m_beacons, m_pending, m_expired_pending);
+    int height = m_beacon_db.Initialize(m_beacons, m_pending, m_expired_pending, m_beacon_first_entries, false);
 
     LogPrint(LogFlags::BEACON, "INFO: %s: m_beacon_db size after load: %u", __func__, m_beacon_db.size());
     LogPrint(LogFlags::BEACON, "INFO: %s: m_beacons size after load: %u", __func__, m_beacons.size());

src/gridcoin/beacon.h

@@ -830,6 +830,8 @@ class BeaconRegistry : public IContractHandler
     //!
     std::set<Beacon_ptr> m_expired_pending;
 
+    BeaconMap m_beacon_first_entries {}; //!< Not used here. Only to satisfy the template.
+
     //!
     //! \brief The member variable that is the instance of the beacon database. This is private to the
     //! beacon registry and is only accessible by beacon registry functions.

src/gridcoin/contract/registry_db.h

@@ -66,10 +66,14 @@ class RegistryDB
     //! specialization.
     //! \param expired_entries. The map of expired pending entries. This is not used in the general template, only in the
     //! beacons specialization.
+    //! \param first_entries: The map of the first entry for the given key. This is only currently used for the whitelist
+    //! (projects).
+    //!
+    //! \param populate_first_entries. This is a boolean that controls whether the first entries map is populated.
     //!
     //! \return block height up to and including which the entry records were stored.
     //!
-    int Initialize(M& entries, P& pending_entries, X& expired_entries)
+    int Initialize(M& entries, P& pending_entries, X& expired_entries, M& first_entries, const bool& populate_first_entries)
     {
         bool status = true;
         int height = 0;
@@ -173,8 +177,8 @@ class RegistryDB
             m_historical[iter.second.m_hash] = std::make_shared<E>(entry);
             entry_ptr& historical_entry_ptr = m_historical[iter.second.m_hash];
 
-            HandleCurrentHistoricalEntries(entries, pending_entries, expired_entries, entry,
-                                                historical_entry_ptr, recnum, key_type);
+            HandleCurrentHistoricalEntries(entries, pending_entries, expired_entries, first_entries, entry,
+                                                historical_entry_ptr, recnum, key_type, populate_first_entries);
 
             number_passivated += (uint64_t) HandlePreviousHistoricalEntries(historical_entry_ptr);
         } // storage_by_record_num iteration
@@ -198,14 +202,16 @@ class RegistryDB
     //!
     //! \param entries
     //! \param pending_entries
+    //! \param expired_entries
+    //! \param first_entries
     //! \param entry
     //! \param historical_entry_ptr
     //! \param recnum
     //! \param key_type
     //!
-    void HandleCurrentHistoricalEntries(M& entries, P& pending_entries, X& expired_entries, const E& entry,
+    void HandleCurrentHistoricalEntries(M& entries, P& pending_entries, X& expired_entries, M& first_entries, const E& entry,
                                         entry_ptr& historical_entry_ptr, const uint64_t& recnum,
-                                        const std::string& key_type)
+                                        const std::string& key_type, const bool& populate_first_entries)
     {
         // The unknown or out of bound status conditions should have never made it into leveldb to begin with, since
         // the entry contract will fail validation, but to be thorough, include the filter condition anyway.
@@ -226,6 +232,10 @@ class RegistryDB
 
             // Insert or replace the existing map entry with the latest.
             entries[entry.Key()] = historical_entry_ptr;
+
+            if (populate_first_entries && historical_entry_ptr->m_previous_hash.IsNull()) {
+                first_entries[entry.Key()] = historical_entry_ptr;
+            }
         }
     }
 

src/gridcoin/project.cpp

@@ -369,18 +369,25 @@ void AutoGreylist::RefreshWithSuperblock(SuperblockPtr superblock_ptr_in)
 {
     LOCK(lock);
 
+    if (superblock_ptr_in.IsEmpty()) {
+        return;
+    }
+
     // We need the current whitelist, including all records except deleted. This will include greylisted projects.
     // NOTE that the refresh_greylist is set to false here and MUST be this when called in the AutoGreylist class itself,
     // to avoid an infinite loop; include_override is also set to false because each refresh of the auto greylist must start
     // with the underlying whitelist state.
     const WhitelistSnapshot whitelist = GetWhitelist().Snapshot(GRC::ProjectEntry::ProjectFilterFlag::ALL_BUT_DELETED, false, false);
 
-    m_greylist_ptr->clear();
+     m_greylist_ptr->clear();
 
+    // No need to go further if the whitelist is empty (ignoring deleted records).
     if (!whitelist.Populated()) {
         return;
     }
 
+    const Whitelist::ProjectEntryMap& project_first_actives = GetWhitelist().GetProjectsFirstActive();
+
     unsigned int v3_superblock_count = 0;
 
     if (superblock_ptr_in->m_version > 2) {
@@ -392,16 +399,26 @@ void AutoGreylist::RefreshWithSuperblock(SuperblockPtr superblock_ptr_in)
     for (const auto& iter : whitelist) {
         auto project = superblock_ptr_in->m_projects_all_cpids_total_credits.m_projects_all_cpid_total_credits.find(iter.m_name);
 
-        if (project != superblock_ptr_in->m_projects_all_cpids_total_credits.m_projects_all_cpid_total_credits.end()) {
-            // Record new greylist candidate entry baseline with the total credit for each project present in superblock.
-            m_greylist_ptr->insert(std::make_pair(iter.m_name,
-                                                  GreylistCandidateEntry(iter.m_name, std::nearbyint(project->second))));
-        } else {
-            // Record new greylist candidate entry with nullopt total credit. This is for a project that is in the whitelist,
-            // but does not have a project entry in the superblock. This would be because the scrapers could not converge on the
-            // project.
-            m_greylist_ptr->insert(std::make_pair(iter.m_name, GreylistCandidateEntry(iter.m_name,
-                                                                                      std::optional<uint64_t>(std::nullopt))));
+        // This record MUST be found, because for the record to be in the whitelist, it must have at least a first record.
+        auto project_first_active = project_first_actives.find(iter.m_name);
+
+        // The purpose of this time comparison is to ONLY post greylist candidate entry (updates) for superblocks that are equal
+        // to or after the first entry date. Remember we are going backwards here. There cannot be entries held against a
+        // whitelisted project from before it was ever whitelisted. This check is required to ensure the greylist rules work
+        // correctly for newly whitelisted projects that are within the 40 day window for WAS and 20 day window for ZCD.
+        if (project_first_active != project_first_actives.end()
+            && superblock_ptr_in.m_timestamp >= project_first_active->second->m_timestamp) {
+            if (project != superblock_ptr_in->m_projects_all_cpids_total_credits.m_projects_all_cpid_total_credits.end()) {
+                // Record new greylist candidate entry baseline with the total credit for each project present in superblock.
+                m_greylist_ptr->insert(std::make_pair(iter.m_name,
+                                                      GreylistCandidateEntry(iter.m_name, std::nearbyint(project->second))));
+            } else {
+                // Record new greylist candidate entry with nullopt total credit. This is for a project that is in the whitelist,
+                // but does not have a project entry in the superblock. This would be because the scrapers could not converge on the
+                // project.
+                m_greylist_ptr->insert(std::make_pair(iter.m_name, GreylistCandidateEntry(iter.m_name,
+                                                                                          std::optional<uint64_t>(std::nullopt))));
+            }
         }
     }
 
@@ -432,6 +449,7 @@ void AutoGreylist::RefreshWithSuperblock(SuperblockPtr superblock_ptr_in)
         }
 
         SuperblockPtr superblock_ptr = block.GetClaim().m_superblock;
+        superblock_ptr.Rebind(index_ptr);
 
         if (superblock_ptr->m_version > 2) {
             for (const auto& iter : whitelist) {
@@ -440,14 +458,24 @@ void AutoGreylist::RefreshWithSuperblock(SuperblockPtr superblock_ptr_in)
 
                 auto project = superblock_ptr->m_projects_all_cpids_total_credits.m_projects_all_cpid_total_credits.find(iter.m_name);
 
-                if (project != superblock_ptr->m_projects_all_cpids_total_credits.m_projects_all_cpid_total_credits.end()) {
-                    // Update greylist candidate entry with the total credit for each project present in superblock.
-                    greylist_entry->second.UpdateGreylistCandidateEntry(project->second, superblock_count);
-                } else {
-                    // Record updated greylist candidate entry with nullopt total credit. This is for a project that is in the whitelist,
-                    // but does not have a project entry in this superblock. This would be because the scrapers could not converge on the
-                    // project for this superblock.
-                    greylist_entry->second.UpdateGreylistCandidateEntry(std::optional<uint64_t>(std::nullopt), superblock_count);
+                // This record MUST be found, because for the record to be in the whitelist, it must have at least a first record.
+                auto project_first_active = project_first_actives.find(iter.m_name);
+
+                // The purpose of this time comparison is to ONLY post greylist candidate entry (updates) for superblocks that are
+                // equal to or after the first entry date. Remember we are going backwards here. There cannot be entries held against
+                // a whitelisted project from before it was ever whitelisted. This check is required to ensure the greylist rules work
+                // correctly for newly whitelisted projects that are within the 40 day window for WAS and 20 day window for ZCD.
+                if (project_first_active != project_first_actives.end()
+                    && superblock_ptr.m_timestamp >= project_first_active->second->m_timestamp) {
+                    if (project != superblock_ptr->m_projects_all_cpids_total_credits.m_projects_all_cpid_total_credits.end()) {
+                        // Update greylist candidate entry with the total credit for each project present in superblock.
+                        greylist_entry->second.UpdateGreylistCandidateEntry(project->second, superblock_count);
+                    } else {
+                        // Record updated greylist candidate entry with nullopt total credit. This is for a project that is in the
+                        // whitelist, but does not have a project entry in this superblock. This would be because the scrapers could
+                        // not converge on the project for this superblock.
+                        greylist_entry->second.UpdateGreylistCandidateEntry(std::optional<uint64_t>(std::nullopt), superblock_count);
+                    }
                 }
             }
 
@@ -615,6 +643,9 @@ void Whitelist::Reset()
     LOCK(cs_lock);
 
     m_project_entries.clear();
+    m_pending_project_entries.clear();
+    m_expired_project_entries.clear();
+    m_project_first_actives.clear();
     m_project_db.clear();
 }
 
@@ -657,14 +688,22 @@ void Whitelist::AddDelete(const ContractContext& ctx)
     // Is there an existing project entry in the map?
     bool current_project_entry_present = (project_entry_pair_iter != m_project_entries.end());
 
-    // If so, then get a smart pointer to it.
+    // If so, then get a smart pointer to it. If not then place an entry in the m_project_first_actives map.
     if (current_project_entry_present) {
         current_project_entry_ptr = project_entry_pair_iter->second;
 
         // Set the payload m_entry's prev entry ctx hash = to the existing entry's hash.
         payload.m_previous_hash = current_project_entry_ptr->m_hash;
     } else { // Original entry for this project entry key
         payload.m_previous_hash = uint256 {};
+
+        // We do not need to check whether the status is ACTIVE, as the first record will be active by the above.
+        //
+        // The downside of this is that this map will hold a reference to each project that was ever active and they will
+        // be ineligible for passivation as a result. However, the memory use for this is minimal given the relatively
+        // small number of projects. It is worth it given the alternative of having to walk each project chainlet to determine
+        // first active for every update of the auto greylist.
+        m_project_first_actives.insert(std::make_pair(current_project_entry_ptr->m_name, current_project_entry_ptr));
     }
 
     LogPrint(LogFlags::CONTRACT, "INFO: %s: project entry add/delete: contract m_version = %u, payload "
@@ -786,6 +825,15 @@ void Whitelist::Revert(const ContractContext& ctx)
         }
 
         if (resurrect_hash.IsNull()) {
+            // The reverted record was the first record, so we need to revert (remove) the entry in the
+            // m_project_first_actives map.
+            if (!m_project_first_actives.erase(key)) {
+                error("%S: The project first active entry for project \"%s\" in the first actives map was not found "
+                      "to erase during a revert. This condition should not occur.",
+                      __func__,
+                      key);
+            }
+
             return;
         }
 
@@ -837,7 +885,11 @@ int Whitelist::Initialize()
 {
     LOCK(cs_lock);
 
-    int height = m_project_db.Initialize(m_project_entries, m_pending_project_entries, m_expired_project_entries);
+    int height = m_project_db.Initialize(m_project_entries,
+                                         m_pending_project_entries,
+                                         m_expired_project_entries,
+                                         m_project_first_actives,
+                                         true);
 
     LogPrint(LogFlags::CONTRACT, "INFO: %s: m_project_db size after load: %u", __func__, m_project_db.size());
     LogPrint(LogFlags::CONTRACT, "INFO: %s: m_project_entries size after load: %u", __func__, m_project_entries.size());
@@ -868,6 +920,9 @@ void Whitelist::ResetInMemoryOnly()
     LOCK(cs_lock);
 
     m_project_entries.clear();
+    m_pending_project_entries.clear();
+    m_expired_project_entries.clear();
+    m_project_first_actives.clear();
     m_project_db.clear_in_memory_only();
 }
 
@@ -878,6 +933,13 @@ uint64_t Whitelist::PassivateDB()
     return m_project_db.passivate_db();
 }
 
+const Whitelist::ProjectEntryMap Whitelist::GetProjectsFirstActive() const
+{
+    LOCK(cs_lock);
+
+    return m_project_first_actives;
+}
+
 Whitelist::ProjectEntryDB &Whitelist::GetProjectDB()
 {
     return m_project_db;

src/gridcoin/project.h

@@ -839,6 +839,12 @@ class Whitelist : public IContractHandler
     //!
     uint64_t PassivateDB() override;
 
+    //!
+    //! \brief This returns m_project_first_actives.
+    //! \return
+    //!
+    const ProjectEntryMap GetProjectsFirstActive() const;
+
     //!
     //! \brief Specializes the template RegistryDB for the ScraperEntry class. Note that std::set<ProjectEntry> is not
     //! actually used.
@@ -875,7 +881,9 @@ class Whitelist : public IContractHandler
 
     std::set<ProjectEntry> m_expired_project_entries {}; //!< Not actually used. Only to satisfy the template.
 
-    ProjectEntryDB m_project_db; //!< The project db member
+    ProjectEntryMap m_project_first_actives;             //!< Tracks when projects were first activated for auto greylisting purposes.
+
+    ProjectEntryDB m_project_db;                         //!< The project db member
 public:
 
     ProjectEntryDB& GetProjectDB();

src/gridcoin/protocol.cpp

@@ -482,7 +482,11 @@ int ProtocolRegistry::Initialize()
 {
     LOCK(cs_lock);
 
-    int height = m_protocol_db.Initialize(m_protocol_entries, m_pending_protocol_entries, m_expired_protocol_entries);
+    int height = m_protocol_db.Initialize(m_protocol_entries,
+                                          m_pending_protocol_entries,
+                                          m_expired_protocol_entries,
+                                          m_protocol_first_entries,
+                                          false);
 
     LogPrint(LogFlags::CONTRACT, "INFO: %s: m_protocol_db size after load: %u", __func__, m_protocol_db.size());
     LogPrint(LogFlags::CONTRACT, "INFO: %s: m_protocol_entries size after load: %u", __func__, m_protocol_entries.size());

src/gridcoin/protocol.h

@@ -592,6 +592,8 @@ class ProtocolRegistry : public IContractHandler
 
     std::set<ProtocolEntry> m_expired_protocol_entries {}; //!< Not used. Only to satisfy the template.
 
+    ProtocolEntryMap m_protocol_first_entries {};          //!< Not used. Only to satisfy the template.
+
     ProtocolEntryDB m_protocol_db;
 
 public:

src/gridcoin/scraper/scraper_registry.cpp

@@ -521,7 +521,7 @@ int ScraperRegistry::Initialize()
 {
     LOCK(cs_lock);
 
-    int height = m_scraper_db.Initialize(m_scrapers, m_pending_scrapers, m_expired_scraper_entries);
+    int height = m_scraper_db.Initialize(m_scrapers, m_pending_scrapers, m_expired_scraper_entries, m_first_scraper_entries, false);
 
     LogPrint(LogFlags::SCRAPER, "INFO: %s: m_scraper_db size after load: %u", __func__, m_scraper_db.size());
     LogPrint(LogFlags::SCRAPER, "INFO: %s: m_scrapers size after load: %u", __func__, m_scrapers.size());

src/gridcoin/scraper/scraper_registry.h

@@ -627,6 +627,8 @@ class ScraperRegistry : public IContractHandler
 
     std::set<ScraperEntry> m_expired_scraper_entries {}; //!< Not actually used for scrapers. To satisfy the template only.
 
+    ScraperMap m_first_scraper_entries {};   //!< Not used here. To satisfy the template only.
+
     ScraperEntryDB m_scraper_db;
 
 public: