[narwhal] Use CertificateV2 to speed up narwhal catchup (#13985)

## Description 

[PR#13777](#13777) introduces new
`SignatureVerificationState` that can now be used in certificate
fetching to only verify the tip of certificate chains and ensure that
the verification state is reflected in storage. Doing so should save us
time in signature verification for a node that is trying to catchup.

## Test Plan 

Added unit tests. Catchup tests in private-testnet.

## Results 

#### NW Catchup Rate @ [200 TPS w/ 2 hours of
downtime](https://metrics.sui.io/d/ORCQSHfVk/nw-catchup-dashboard?var-Environment=8Xt1pVoVk&var-network=private-testnet&var-validator=ams-ptn-val-00&var-validator=ams-ptn-val-09&orgId=1&from=1696385905867&to=1696388154869)
![Screenshot 2023-10-10 at 4 25 53
PM](https://github.com/MystenLabs/sui/assets/97870774/ae71c43d-ec24-4d56-85f5-0703c9794ef8)


#### NW Catchup Rate @ [5K TPS w/ 1 hour of
downtime](https://metrics.sui.io/d/ORCQSHfVk/nw-catchup-dashboard?var-Environment=8Xt1pVoVk&var-network=private-testnet&var-validator=ams-ptn-val-00&var-validator=ams-ptn-val-02&var-validator=ams-ptn-val-09&var-validator=del-ptn-val-08&var-validator=sjc-ptn2-val-00&var-validator=ams-ptn-val-03&orgId=1&from=1696643114770&to=1696651260983)
![Screenshot 2023-10-10 at 4 26 24
PM](https://github.com/MystenLabs/sui/assets/97870774/1e5531eb-9152-4334-a715-a1d6136dd804)

## Known Issues to be investigated/fixed in follow up PRs

#### Narwhal catchup only hits full potential after state sync completes
![Screenshot 2023-10-10 at 4 31 21
PM](https://github.com/MystenLabs/sui/assets/97870774/23f25ffe-0998-40c1-af14-ab6e864b8b8d)

#### Execution bottleneck at high TPS preventing higher NW catchup rate
![Screenshot 2023-10-10 at 4 33 39
PM](https://github.com/MystenLabs/sui/assets/97870774/aa222598-4570-42a0-85ad-c2f46c77aba2)


---
If your changes are not user-facing and not a breaking change, you can
skip the following section. Otherwise, please indicate what changed, and
then add to the Release Notes section as highlighted during the release
process.

### Type of Change (Check all that apply)

- [X] protocol change
- [ ] user-visible impact
- [ ] breaking change for a client SDKs
- [ ] breaking change for FNs (FN binary must upgrade)
- [ ] breaking change for validators or node operators (must upgrade
binaries)
- [ ] breaking change for on-chain data layout
- [ ] necessitate either a data wipe or data migration

### Release notes

Protocol upgrade to version 28 which will enable the use of
CertificateV2 in narwhal that will be used to speed up processing of
certificates during certificate fetching/catchup.

crates/sui-protocol-config/src/lib.rs

@@ -76,7 +76,8 @@ const MAX_PROTOCOL_VERSION: u64 = 28;
 // Version 25: Add sui::table_vec::swap and sui::table_vec::swap_remove to system packages.
 // Version 26: New gas model version.
 //             Add support for receiving objects off of other objects in devnet only.
-// Version 27: Add sui::zklogin::verify_zklogin_id and related functions to sui framework.
+// Version 28: Add sui::zklogin::verify_zklogin_id and related functions to sui framework.
+//             Use CertificateV2 in narwhal
 
 #[derive(Copy, Clone, Debug, Hash, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
 pub struct ProtocolVersion(u64);
@@ -1529,9 +1530,10 @@ impl ProtocolConfig {
                     cfg.check_zklogin_id_cost_base = Some(200);
                     // zklogin::check_zklogin_issuer
                     cfg.check_zklogin_issuer_cost_base = Some(200);
-                    // Only enable effects v2 on devnet.
+                    // Only enable effects v2 & nw certificate v2 on devnet.
                     if chain != Chain::Mainnet && chain != Chain::Testnet {
                         cfg.feature_flags.enable_effects_v2 = true;
+                        cfg.feature_flags.narwhal_certificate_v2 = true;
                     }
                 }
                 // Use this template when making changes:

crates/sui-protocol-config/src/snapshots/sui_protocol_config__test__version_28.snap

@@ -36,6 +36,7 @@ feature_flags:
   loaded_child_object_format_type: true
   receive_objects: true
   enable_effects_v2: true
+  narwhal_certificate_v2: true
 max_tx_size_bytes: 131072
 max_input_objects: 2048
 max_size_written_objects: 5000000

narwhal/node/tests/staged/narwhal.yaml

@@ -32,23 +32,21 @@ BatchV2:
         TYPENAME: VersionedMetadata
 Certificate:
   ENUM:
-    0:
-      V1:
+    1:
+      V2:
         NEWTYPE:
-          TYPENAME: CertificateV1
+          TYPENAME: CertificateV2
 CertificateDigest:
   NEWTYPESTRUCT:
     TUPLEARRAY:
       CONTENT: U8
       SIZE: 32
-CertificateV1:
+CertificateV2:
   STRUCT:
     - header:
         TYPENAME: Header
-    - aggregated_signature:
-        TUPLEARRAY:
-          CONTENT: U8
-          SIZE: 48
+    - signature_verification_state:
+        TYPENAME: SignatureVerificationState
     - signed_authorities: BYTES
     - metadata:
         TYPENAME: Metadata
@@ -88,6 +86,14 @@ MetadataV1:
     - created_at: U64
     - received_at:
         OPTION: U64
+SignatureVerificationState:
+  ENUM:
+    0:
+      Unsigned:
+        NEWTYPE:
+          TUPLEARRAY:
+            CONTENT: U8
+            SIZE: 48
 VersionedMetadata:
   ENUM:
     0:

narwhal/primary/src/aggregators.rs

@@ -16,7 +16,7 @@ use tracing::warn;
 use types::{
     ensure,
     error::{DagError, DagResult},
-    Certificate, CertificateAPI, Header, Vote, VoteAPI,
+    Certificate, CertificateAPI, Header, SignatureVerificationState, Vote, VoteAPI,
 };
 
 /// Aggregates votes for a particular header into a certificate.
@@ -62,7 +62,7 @@ impl VotesAggregator {
             .votes_received_last_round
             .set(self.votes.len() as i64);
         if self.weight >= committee.quorum_threshold() {
-            let cert = Certificate::new_unverified(
+            let mut cert = Certificate::new_unverified(
                 &self.protocol_config,
                 committee,
                 header.clone(),
@@ -83,24 +83,34 @@ impl VotesAggregator {
                         "Failed to verify aggregated sig on certificate: {} error: {}",
                         certificate_digest, err
                     );
-                    let mut i = 0;
-                    while i < self.votes.len() {
-                        let (id, sig) = &self.votes[i];
+                    self.votes.retain(|(id, sig)| {
                         let pk = committee.authority_safe(id).protocol_key();
                         if sig
                             .verify_secure(&to_intent_message(certificate_digest), pk)
                             .is_err()
                         {
                             warn!("Invalid signature on header from authority: {}", id);
                             self.weight -= committee.stake(pk);
-                            self.votes.remove(i);
+                            false
                         } else {
-                            i += 1;
+                            true
                         }
-                    }
+                    });
                     return Ok(None);
                 }
-                Ok(_) => return Ok(Some(cert)),
+                Ok(_) => {
+                    // TODO: Move this block and the AggregateSignature verification into Certificate
+                    if self.protocol_config.narwhal_certificate_v2() {
+                        cert.set_signature_verification_state(
+                            SignatureVerificationState::VerifiedDirectly(
+                                cert.aggregated_signature()
+                                    .ok_or(DagError::InvalidSignature)?
+                                    .clone(),
+                            ),
+                        );
+                    }
+                    return Ok(Some(cert));
+                }
             }
         }
         Ok(None)