Update handle allowed set and add new handles RPC (#2239)

# Goal
The goal of this PR is to remove some undesirable characters from the
handle allowed list and create new RPCs to make testing handles (and the
edge cases) easier

Closes #2237

# Discussion

- Removing some punctuation only characters from the allowed list
- Added a new `check_handle` RPC so that it is easier to check the
canonical value along with validity and if there are suffixes remaining

# Checklist
- [x] Updated js/api-augment for Custom RPC APIs?
- [x] Unit Tests added?
- [x] e2e Tests added?
- [x] Spec version incremented?

common/primitives/src/handles.rs

@@ -94,3 +94,33 @@ pub struct PresumptiveSuffixesResponse {
 	/// The suffixes
 	pub suffixes: Vec<HandleSuffix>,
 }
+
+/// Output response for retrieving the next suffixes for a given handle
+#[cfg_attr(feature = "std", derive(Serialize, Deserialize))]
+#[derive(Clone, Encode, Decode, PartialEq, Debug, TypeInfo, Eq)]
+pub struct CheckHandleResponse {
+	/// The base handle
+	#[cfg_attr(feature = "std", serde(with = "as_string"))]
+	pub base_handle: Vec<u8>,
+	/// The canonical handle
+	#[cfg_attr(feature = "std", serde(with = "as_string"))]
+	pub canonical_base: Vec<u8>,
+	/// The current suffix index
+	pub suffix_index: u16,
+	/// Are additional suffixes available?
+	pub suffixes_available: bool,
+	/// Validity
+	pub valid: bool,
+}
+
+impl Default for CheckHandleResponse {
+	fn default() -> Self {
+		Self {
+			base_handle: Vec::new(),
+			canonical_base: Vec::new(),
+			suffix_index: 0,
+			suffixes_available: false,
+			valid: false,
+		}
+	}
+}

e2e/handles/handles.test.ts

@@ -119,6 +119,10 @@ describe('🤝 Handles', function () {
         assert(msaOption.isSome, 'msaOption should be Some');
         const msaFromHandle = msaOption.unwrap();
         assert.equal(msaFromHandle.toString(), msa_id.toString(), 'msaFromHandle should be equal to msa_id');
+
+        // Check that the rpc returns the index as > 0
+        const apiCheck = await ExtrinsicHelper.apiPromise.call.handlesRuntimeApi.checkHandle(handle);
+        assert(apiCheck.suffixIndex.toNumber() > 0);
       });
     });
 
@@ -166,4 +170,42 @@ describe('🤝 Handles', function () {
       assert.equal(res.toHuman(), false);
     });
   });
+
+  describe('checkHandle basic test', function () {
+    it('expected outcome for a good handle', async function () {
+      const res = await ExtrinsicHelper.apiPromise.call.handlesRuntimeApi.checkHandle('Little Bobby Tables');
+      assert(!res.isEmpty, 'Expected a response');
+      assert.deepEqual(res.toHuman(), {
+        baseHandle: 'Little Bobby Tables',
+        canonicalBase: 'l1tt1eb0bbytab1es',
+        suffixIndex: '0',
+        suffixesAvailable: true,
+        valid: true,
+      });
+    });
+
+    it('expected outcome for a bad handle', async function () {
+      const res = await ExtrinsicHelper.apiPromise.call.handlesRuntimeApi.checkHandle('Robert`DROP TABLE STUDENTS;--');
+      assert(!res.isEmpty, 'Expected a response');
+      assert.deepEqual(res.toHuman(), {
+        baseHandle: 'Robert`DROP TABLE STUDENTS;--',
+        canonicalBase: '',
+        suffixIndex: '0',
+        suffixesAvailable: false,
+        valid: false,
+      });
+    });
+
+    it('expected outcome for a good handle with complex whitespace', async function () {
+      const res = await ExtrinsicHelper.apiPromise.call.handlesRuntimeApi.checkHandle('नी हुन्‍न् ।');
+      assert(!res.isEmpty, 'Expected a response');
+      assert.deepEqual(res.toHuman(), {
+        baseHandle: '0xe0a4a8e0a58020e0a4b9e0a581e0a4a8e0a58de2808de0a4a8e0a58d20e0a5a4',
+        canonicalBase: '0xe0a4a8e0a4b9e0a4a8e0a4a8e0a5a4',
+        suffixIndex: '0',
+        suffixesAvailable: true,
+        valid: true,
+      });
+    });
+  });
 });

pallets/capacity/src/runtime-api/src/lib.rs

@@ -27,7 +27,6 @@ use sp_std::vec::Vec;
 sp_api::decl_runtime_apis! {
 	/// Runtime Version for Capacity
 	/// - MUST be incremented if anything changes
-	/// - Also update in js/api-augment
 	/// - See: https://paritytech.github.io/polkadot/doc/polkadot_primitives/runtime_api/index.html
 	#[api_version(1)]
 	/// Runtime APIs for [Capacity](../pallet_capacity/index.html)

pallets/handles/src/handles-utils/constants.rs

@@ -3,6 +3,7 @@
 use core::ops::RangeInclusive;
 
 #[cfg(test)]
+#[allow(dead_code)]
 pub fn build_allowed_char_ranges() -> Vec<RangeInclusive<u16>> {
 	let mut new_allowed: Vec<RangeInclusive<u16>> = Vec::new();
 	let mut last: RangeInclusive<u16> = RangeInclusive::new(0u16, 0u16);
@@ -33,7 +34,7 @@ pub fn build_allowed_char_ranges() -> Vec<RangeInclusive<u16>> {
 #[rustfmt::skip]
 pub const ALLOWED_UNICODE_CHARACTER_RANGES: [RangeInclusive<u16>; 54] = [
 0x0020..=0x007A,
-0x0080..=0x0024F,
+0x0080..=0x024F,
 0x02B0..=0x04FF,
 0x0531..=0x058A,
 0x0591..=0x05F4,
@@ -70,7 +71,7 @@ pub const ALLOWED_UNICODE_CHARACTER_RANGES: [RangeInclusive<u16>; 54] = [
 0x1B80..=0x1BB9,
 0x1BC0..=0x1C7F,
 0x1E00..=0x1FFF,
-0x200C..=0x206F,
+0x200C..=0x200D,
 0x2C80..=0x2CFF,
 0x2D30..=0x2D7F,
 0x3040..=0x30FF,
@@ -146,7 +147,7 @@ pub const ALLOWED_UNICODE_CHARACTER_RANGES: [RangeInclusive<u16>; 54] = [
 //     0x1C50..=0x1C7F, // Ol Chiki
 //     0x1E00..=0x1EFF, // Latin Extended Additional
 //     0x1F00..=0x1FFF, // Greek Extended
-//     0x200C..=0x206F, // General punctuation, used in some languages to indicate syllables such as glottal stops
+//     0x200C..=0x200D, // General punctuation Limited to the Zero-width Joiners
 //     0x2C80..=0x2CFF, // Coptic
 //     0x2D30..=0x2D7F, // Tifinagh
 //     0x3040..=0x309F, // Hiragana

pallets/handles/src/handles-utils/src/converter.rs

@@ -105,7 +105,11 @@ pub fn split_display_name(display_name_str: &str) -> Option<(String, HandleSuffi
 pub fn strip_unicode_whitespace(input_str: &str) -> String {
 	input_str
 		.chars()
-		.filter(|character| !character.is_whitespace())
+		// U+200C is a zero-width Non-joiner needed for some writing systems
+		// U+200D is a zero-width joiner needed for some writing systems
+		.filter(|character| {
+			!character.is_whitespace() && character.ne(&'\u{200C}') && character.ne(&'\u{200D}')
+		})
 		.collect::<alloc::string::String>()
 }
 

pallets/handles/src/handles-utils/src/tests/converter_tests.rs

@@ -77,6 +77,8 @@ fn test_strip_unicode_whitespace() {
 		'\u{202F}', // narrow no-break space
 		'\u{205F}', // medium mathematical space
 		'\u{3000}', // ideographic space
+		'\u{200C}', // Zero-width Non Joiner
+		'\u{200D}', // Zero-width Joiner
 	];
 	let whitespace_string: String = whitespace_chars.into_iter().collect();
 	let string_with_whitespace =

pallets/handles/src/handles-utils/src/tests/validator_tests.rs

@@ -48,6 +48,8 @@ fn test_contains_blocked_characters_negative() {
 // Some translations: https://translate.glosbe.com/
 // Others from Wikipedia
 // Many are (supposed to be) common names or greetings, or translations of "beautiful flower"
+// Helpful tool to convert strings into points for testing:
+//     https://onlinetools.com/utf8/convert-utf8-to-code-points
 #[rustfmt::skip]
 #[test]
 fn test_consists_of_supported_unicode_character_sets_happy_path() {

pallets/handles/src/lib.rs

@@ -567,6 +567,53 @@ pub mod pallet {
 			PresumptiveSuffixesResponse { base_handle: base_handle.into(), suffixes }
 		}
 
+		/// Check a base handle for validity and collect information on it
+		///
+		/// This function takes a `Vec<u8>` handle and checks to make sure it is:
+		/// - Valid
+		/// - Has suffixes remaining
+		///
+		/// It also returns the original input as well as the canonical version
+		///
+		/// # Arguments
+		///
+		/// * `handle` - The handle to check.
+		///
+		/// # Returns
+		///
+		/// * `CheckHandleResponse`
+		///
+		pub fn check_handle(base_handle: Vec<u8>) -> CheckHandleResponse {
+			let valid = Self::validate_handle(base_handle.to_vec());
+
+			if !valid {
+				return CheckHandleResponse {
+					base_handle: base_handle.into(),
+					..Default::default()
+				};
+			}
+
+			let base_handle_str = core::str::from_utf8(&base_handle).unwrap_or_default();
+
+			// Convert base handle into a canonical base
+			let (_canonical_handle_str, canonical_base) =
+				Self::get_canonical_string_vec_from_base_handle(&base_handle_str);
+
+			let suffix_index =
+				Self::get_next_suffix_index_for_canonical_handle(canonical_base.clone())
+					.unwrap_or_default();
+
+			let suffixes_available = suffix_index < T::HandleSuffixMax::get();
+
+			CheckHandleResponse {
+				base_handle: base_handle.into(),
+				suffix_index,
+				suffixes_available,
+				valid,
+				canonical_base: canonical_base.into(),
+			}
+		}
+
 		/// Retrieve a `MessageSourceId` for a given handle.
 		///
 		/// # Arguments

pallets/handles/src/runtime-api/src/lib.rs

@@ -18,7 +18,9 @@
 //! - Runtime interfaces for end users beyond just State Queries
 
 use common_primitives::{
-	handles::{BaseHandle, DisplayHandle, HandleResponse, PresumptiveSuffixesResponse},
+	handles::{
+		BaseHandle, CheckHandleResponse, DisplayHandle, HandleResponse, PresumptiveSuffixesResponse,
+	},
 	msa::MessageSourceId,
 };
 
@@ -28,7 +30,6 @@ sp_api::decl_runtime_apis! {
 
 	/// Runtime Version for Handles
 	/// - MUST be incremented if anything changes
-	/// - Also update in js/api-augment
 	/// - See: https://paritytech.github.io/polkadot/doc/polkadot_primitives/runtime_api/index.html
 	#[api_version(2)]
 
@@ -46,5 +47,9 @@ sp_api::decl_runtime_apis! {
 
 		/// Check if a handle is valid
 		fn validate_handle(base_handle: BaseHandle) -> bool;
+
+		#[api_version(3)]
+		/// Return information about a given handle
+		fn check_handle(base_handle: BaseHandle) -> CheckHandleResponse;
 	}
 }

pallets/handles/src/tests/handle_creation_tests.rs

@@ -1,5 +1,8 @@
 use crate::{tests::mock::*, Error, Event};
-use common_primitives::{handles::HANDLE_BYTES_MAX, msa::MessageSourceId};
+use common_primitives::{
+	handles::{CheckHandleResponse, HANDLE_BYTES_MAX},
+	msa::MessageSourceId,
+};
 use frame_support::{assert_err, assert_noop, assert_ok, dispatch::DispatchResult};
 use parity_scale_codec::Decode;
 use sp_core::{sr25519, Encode, Pair};
@@ -420,3 +423,39 @@ fn test_validate_handle() {
 		assert_eq!(Handles::validate_handle(handle_with_emoji.as_bytes().to_vec()), false);
 	})
 }
+
+#[test]
+fn test_check_handle() {
+	new_test_ext().execute_with(|| {
+		let good_handle: String = String::from("MyBonny");
+		assert_eq!(
+			Handles::check_handle(good_handle.as_bytes().to_vec()),
+			CheckHandleResponse {
+				base_handle: good_handle.as_bytes().to_vec(),
+				suffix_index: 0,
+				suffixes_available: true,
+				valid: true,
+				canonical_base: String::from("myb0nny").as_bytes().to_vec(),
+			}
+		);
+
+		let too_long_handle: String =
+			std::iter::repeat('*').take((HANDLE_BYTES_MAX + 1) as usize).collect();
+		assert_eq!(
+			Handles::check_handle(too_long_handle.as_bytes().to_vec()),
+			CheckHandleResponse {
+				base_handle: too_long_handle.as_bytes().to_vec(),
+				..Default::default()
+			}
+		);
+
+		let handle_with_emoji = format_args!("John{}", '\u{1F600}').to_string();
+		assert_eq!(
+			Handles::check_handle(handle_with_emoji.as_bytes().to_vec()),
+			CheckHandleResponse {
+				base_handle: handle_with_emoji.as_bytes().to_vec(),
+				..Default::default()
+			}
+		);
+	})
+}

pallets/messages/src/runtime-api/src/lib.rs

@@ -26,7 +26,6 @@ sp_api::decl_runtime_apis! {
 
 	/// Runtime Version for Messages
 	/// - MUST be incremented if anything changes
-	/// - Also update in js/api-augment
 	/// - See: https://paritytech.github.io/polkadot/doc/polkadot_primitives/runtime_api/index.html
 	#[api_version(1)]
 

pallets/msa/src/runtime-api/src/lib.rs

@@ -27,7 +27,6 @@ sp_api::decl_runtime_apis! {
 
 	/// Runtime Version for MSAs
 	/// - MUST be incremented if anything changes
-	/// - Also update in js/api-augment
 	/// - See: https://paritytech.github.io/polkadot/doc/polkadot_primitives/runtime_api/index.html
 	#[api_version(2)]
 

pallets/schemas/src/runtime-api/src/lib.rs

@@ -24,7 +24,6 @@ sp_api::decl_runtime_apis! {
 
 	/// Runtime Version for Schemas
 	/// - MUST be incremented if anything changes
-	/// - Also update in js/api-augment
 	/// - See: https://paritytech.github.io/polkadot/doc/polkadot_primitives/runtime_api/index.html
 	#[api_version(2)]
 

pallets/stateful-storage/src/runtime-api/src/lib.rs

@@ -31,7 +31,6 @@ sp_api::decl_runtime_apis! {
 
 	/// Runtime Version for Stateful Storage
 	/// - MUST be incremented if anything changes
-	/// - Also update in js/api-augment
 	/// - See: https://paritytech.github.io/polkadot/doc/polkadot_primitives/runtime_api/index.html
 	#[api_version(1)]
 

runtime/frequency/src/lib.rs

@@ -42,7 +42,9 @@ use sp_version::RuntimeVersion;
 use static_assertions::const_assert;
 
 use common_primitives::{
-	handles::{BaseHandle, DisplayHandle, HandleResponse, PresumptiveSuffixesResponse},
+	handles::{
+		BaseHandle, CheckHandleResponse, DisplayHandle, HandleResponse, PresumptiveSuffixesResponse,
+	},
 	messages::MessageResponse,
 	msa::{
 		DelegationResponse, DelegationValidator, DelegatorId, MessageSourceId, ProviderId,
@@ -399,7 +401,7 @@ pub const VERSION: RuntimeVersion = RuntimeVersion {
 	spec_name: create_runtime_str!("frequency"),
 	impl_name: create_runtime_str!("frequency"),
 	authoring_version: 1,
-	spec_version: 139,
+	spec_version: 140,
 	impl_version: 0,
 	apis: RUNTIME_API_VERSIONS,
 	transaction_version: 1,
@@ -413,7 +415,7 @@ pub const VERSION: RuntimeVersion = RuntimeVersion {
 	spec_name: create_runtime_str!("frequency-testnet"),
 	impl_name: create_runtime_str!("frequency"),
 	authoring_version: 1,
-	spec_version: 139,
+	spec_version: 140,
 	impl_version: 0,
 	apis: RUNTIME_API_VERSIONS,
 	transaction_version: 1,
@@ -1590,6 +1592,7 @@ sp_api::impl_runtime_apis! {
 		}
 	}
 
+	#[api_version(3)]
 	impl pallet_handles_runtime_api::HandlesRuntimeApi<Block> for Runtime {
 		fn get_handle_for_msa(msa_id: MessageSourceId) -> Option<HandleResponse> {
 			Handles::get_handle_for_msa(msa_id)
@@ -1605,7 +1608,11 @@ sp_api::impl_runtime_apis! {
 		fn validate_handle(base_handle: BaseHandle) -> bool {
 			Handles::validate_handle(base_handle.to_vec())
 		}
+		fn check_handle(base_handle: BaseHandle) -> CheckHandleResponse {
+			Handles::check_handle(base_handle.to_vec())
+		}
 	}
+
 	impl pallet_capacity_runtime_api::CapacityRuntimeApi<Block, AccountId, Balance, BlockNumber> for Runtime {
 		fn list_unclaimed_rewards(who: AccountId) -> Vec<UnclaimedRewardInfo<Balance, BlockNumber>> {
 			match Capacity::list_unclaimed_rewards(&who) {

runtime/system-runtime-api/src/lib.rs

@@ -25,7 +25,6 @@ sp_api::decl_runtime_apis! {
 
 	/// Runtime Version for Additional Frequency Runtime Apis
 	/// - MUST be incremented if anything changes
-	/// - Also update in js/api-augment
 	/// - See: https://paritytech.github.io/polkadot/doc/polkadot_primitives/runtime_api/index.html
 	#[api_version(1)]