From d49e8eec43de89b0acef94e8c0fb5ef05507be23 Mon Sep 17 00:00:00 2001
From: Eileen McNaughton <emcnaughton@wikimedia.org>
Date: Thu, 7 Oct 2021 16:58:58 +1300
Subject: [PATCH 1/2] Test & align empty token handling

---
 Civi/Token/TokenCompatSubscriber.php          |  6 +++
 .../CRM/Utils/TokenConsistencyTest.php        | 39 +++++++++++++++++++
 2 files changed, 45 insertions(+)

diff --git a/Civi/Token/TokenCompatSubscriber.php b/Civi/Token/TokenCompatSubscriber.php
index e14539489773..33e394f3cdfc 100644
--- a/Civi/Token/TokenCompatSubscriber.php
+++ b/Civi/Token/TokenCompatSubscriber.php
@@ -60,6 +60,12 @@ public function onRender(TokenRenderEvent $e): void {
       return '';
     });
 
+    // This removes the pattern used in greetings of having bits of text that
+    // depend on the tokens around them - ie '{first_name}{ }{last_name}
+    // has an extra construct '{ }' which will resolve as a space if the
+    // tokens on either side are resolved to 'something'
+    $e->string = preg_replace('/\\\\|\{(\s*)?\}/', ' ', $e->string);
+
     if ($useSmarty) {
       $smartyVars = [];
       foreach ($e->context['smartyTokenAlias'] ?? [] as $smartyName => $tokenName) {
diff --git a/tests/phpunit/CRM/Utils/TokenConsistencyTest.php b/tests/phpunit/CRM/Utils/TokenConsistencyTest.php
index 679cd0b44807..7e7f00c1f14b 100644
--- a/tests/phpunit/CRM/Utils/TokenConsistencyTest.php
+++ b/tests/phpunit/CRM/Utils/TokenConsistencyTest.php
@@ -234,6 +234,45 @@ public function getUnadvertisedTokens(): array {
     ];
   }
 
+  /**
+   * Test tokens in 2 ways to ensure consistent handling.
+   *
+   * 1) as part of the greeting processing
+   * 2) via the token processor.
+   *
+   */
+  public function testOddTokens(): void {
+
+    $variants = [
+      [
+        'string' => '{contact.individual_prefix}{ }{contact.first_name}{ }{contact.middle_name}{ }{contact.last_name}{ }{contact.individual_suffix}',
+        'expected' => 'Mr. Anthony  Anderson II',
+      ],
+      [
+        'string' => '{contact.prefix_id:label}{ }{contact.first_name}{ }{contact.middle_name}{ }{contact.last_name}{ }{contact.suffix_id:label}',
+        'expected' => 'Mr. Anthony  Anderson II',
+      ],
+    ];
+    $tokenProcessor = new TokenProcessor(\Civi::dispatcher(), [
+      'smarty' => FALSE,
+      'schema' => ['contactId'],
+    ]);
+    $contactID = $this->individualCreate(['middle_name' => '']);
+    $tokenProcessor->addRow(['contactId' => $contactID]);
+    $tokenProcessor->evaluate();
+    foreach ($variants as $index => $variant) {
+      $tokenProcessor->addMessage($index, $variant['string'], 'text/plain');
+    }
+    $tokenProcessor->evaluate();
+    $result = $tokenProcessor->getRow(0);
+    foreach ($variants as $index => $variant) {
+      $greetingString = $variant['string'];
+      CRM_Utils_Token::replaceGreetingTokens($greetingString, $this->callAPISuccessGetSingle('Contact', ['id' => $contactID]), $contactID);
+      $this->assertEquals($variant['expected'], $greetingString);
+      $this->assertEquals($variant['expected'], $result->render($index));
+    }
+  }
+
   /**
    * Get the contribution recur tokens keyed by the token.
    *

From 0bb18246a40058b42684fc5fbd23357a8623ecd3 Mon Sep 17 00:00:00 2001
From: Tim Otten <totten@civicrm.org>
Date: Thu, 7 Oct 2021 22:54:51 -0700
Subject: [PATCH 2/2] (NFC) TokenConsistencyTest - Small tweaks

1. You don't need to call evaluate() twice. It should be once after filling both rows+messages.
2. Add messages to clarify assertions
---
 tests/phpunit/CRM/Utils/TokenConsistencyTest.php | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/tests/phpunit/CRM/Utils/TokenConsistencyTest.php b/tests/phpunit/CRM/Utils/TokenConsistencyTest.php
index 7e7f00c1f14b..e5878ab7848e 100644
--- a/tests/phpunit/CRM/Utils/TokenConsistencyTest.php
+++ b/tests/phpunit/CRM/Utils/TokenConsistencyTest.php
@@ -259,7 +259,6 @@ public function testOddTokens(): void {
     ]);
     $contactID = $this->individualCreate(['middle_name' => '']);
     $tokenProcessor->addRow(['contactId' => $contactID]);
-    $tokenProcessor->evaluate();
     foreach ($variants as $index => $variant) {
       $tokenProcessor->addMessage($index, $variant['string'], 'text/plain');
     }
@@ -268,8 +267,8 @@ public function testOddTokens(): void {
     foreach ($variants as $index => $variant) {
       $greetingString = $variant['string'];
       CRM_Utils_Token::replaceGreetingTokens($greetingString, $this->callAPISuccessGetSingle('Contact', ['id' => $contactID]), $contactID);
-      $this->assertEquals($variant['expected'], $greetingString);
-      $this->assertEquals($variant['expected'], $result->render($index));
+      $this->assertEquals($variant['expected'], $greetingString, 'replaceGreetingTokens() should render expected output');
+      $this->assertEquals($variant['expected'], $result->render($index), 'TokenProcessor should render expected output');
     }
   }