feat: Experiment with i18next integration TG-430

packages/core/src/Observer.ts

@@ -1,14 +1,14 @@
 import { CoreHandler } from './handlers/CoreHandler';
 import { Properties } from './Properties';
-import { TextHandler } from './handlers/TextHandler';
+import { InvisibleTextHandler } from './handlers/InvisibleTextHandler';
 import { AttributeHandler } from './handlers/AttributeHandler';
 import { ElementRegistrar } from './services/ElementRegistrar';
 
 export class Observer {
   constructor(
     private properties: Properties,
     private coreHandler: CoreHandler,
-    private basicTextHandler: TextHandler,
+    private basicTextHandler: InvisibleTextHandler,
     private attributeHandler: AttributeHandler,
     private nodeRegistrar: ElementRegistrar
   ) {}

packages/core/src/handlers/AbstractHandler.ts

@@ -86,7 +86,7 @@ export abstract class AbstractHandler {
     return node as Node as NodeWithMeta;
   }
 
-  private lockNode(node: Node | Attr): NodeWithLock {
+  protected lockNode(node: Node | Attr): NodeWithLock {
     if (node[TOLGEE_ATTRIBUTE_NAME] === undefined) {
       node[TOLGEE_ATTRIBUTE_NAME] = {} as NodeLock;
     }
@@ -99,13 +99,13 @@ export abstract class AbstractHandler {
     return node as NodeWithLock;
   }
 
-  private unlockNode(node: Node | Attr) {
+  protected unlockNode(node: Node | Attr) {
     if (node[TOLGEE_ATTRIBUTE_NAME]) {
       node[TOLGEE_ATTRIBUTE_NAME].locked = false;
     }
   }
 
-  private getParentElement(node: Node) {
+  protected getParentElement(node: Node) {
     const parent = this.getSuitableParent(node);
     return AbstractHandler.initParentElement(parent);
   }

packages/core/src/handlers/CoreHandler.ts

@@ -1,6 +1,6 @@
 import { NodeHelper } from '../helpers/NodeHelper';
 import { CoreService } from '../services/CoreService';
-import { TextHandler } from './TextHandler';
+import { InvisibleTextHandler } from './InvisibleTextHandler';
 import { EventService } from '../services/EventService';
 import { Properties } from '../Properties';
 import { AttributeHandler } from './AttributeHandler';
@@ -11,7 +11,7 @@ import { WrappedHandler } from './WrappedHandler';
 export class CoreHandler {
   constructor(
     private service: CoreService,
-    private textHandler: TextHandler,
+    private textHandler: InvisibleTextHandler,
     private eventService: EventService,
     private properties: Properties,
     private attributeHandler: AttributeHandler,

packages/core/src/handlers/InvisibleTextHandler.ts

@@ -0,0 +1,56 @@
+import { NodeHelper } from '../helpers/NodeHelper';
+import { Properties } from '../Properties';
+import { TranslationHighlighter } from '../highlighter/TranslationHighlighter';
+import { TextService } from '../services/TextService';
+import { AbstractHandler } from './AbstractHandler';
+import { ElementRegistrar } from '../services/ElementRegistrar';
+import { INVISIBLE_CHARACTERS } from '../helpers/secret';
+import { TOLGEE_ATTRIBUTE_NAME } from 'Constants/Global';
+import { NodeLock, NodeMeta } from 'types';
+import { InvisibleTextService } from 'services/InvisibleTextService';
+
+export class InvisibleTextHandler extends AbstractHandler {
+  constructor(
+    protected properties: Properties,
+    protected translationHighlighter: TranslationHighlighter,
+    protected textService: TextService,
+    protected invisibleTextService: InvisibleTextService,
+    protected nodeRegistrar: ElementRegistrar
+  ) {
+    super(properties, textService, nodeRegistrar, translationHighlighter);
+  }
+
+  async handle(node: Node): Promise<void> {
+    const xPath = `./descendant-or-self::text()[contains(., '${INVISIBLE_CHARACTERS[0]}')]`;
+    const nodes = NodeHelper.evaluate(xPath, node);
+    const filtered: Text[] = this.filterRestricted(nodes as Text[]);
+
+    await this.handleNodes(filtered);
+  }
+
+  protected async handleNodes(nodes: Array<Text | Attr>) {
+    for (const textNode of nodes) {
+      if (textNode[TOLGEE_ATTRIBUTE_NAME] === undefined) {
+        textNode[TOLGEE_ATTRIBUTE_NAME] = {} as NodeLock;
+      }
+      const tolgeeData = textNode[TOLGEE_ATTRIBUTE_NAME] as
+        | NodeMeta
+        | NodeLock
+        | undefined;
+      if (tolgeeData?.locked !== true) {
+        this.lockNode(textNode);
+        const result = await this.invisibleTextService.replace(
+          textNode.textContent
+        );
+        if (result) {
+          const { text, keys } = result;
+          const translatedNode = this.translateChildNode(textNode, text, keys);
+          const parentElement = this.getParentElement(translatedNode);
+          parentElement._tolgee.nodes.add(translatedNode);
+          this.elementRegistrar.register(parentElement);
+        }
+        this.unlockNode(textNode);
+      }
+    }
+  }
+}

packages/core/src/helpers/encoder.ts

@@ -0,0 +1,96 @@
+// TextEncoder/TextDecoder polyfills for utf-8 - an implementation of TextEncoder/TextDecoder APIs
+// Written in 2013 by Viktor Mukhachev <[email protected]>
+// To the extent possible under law, the author(s) have dedicated all copyright and related and neighboring rights to this software to the public domain worldwide. This software is distributed without any warranty.
+// You should have received a copy of the CC0 Public Domain Dedication along with this software. If not, see <http://creativecommons.org/publicdomain/zero/1.0/>.
+
+// Some important notes about the polyfill below:
+// Native TextEncoder/TextDecoder implementation is overwritten
+// String.prototype.codePointAt polyfill not included, as well as String.fromCodePoint
+// TextEncoder.prototype.encode returns a regular array instead of Uint8Array
+// No options (fatal of the TextDecoder constructor and stream of the TextDecoder.prototype.decode method) are supported.
+// TextDecoder.prototype.decode does not valid byte sequences
+// This is a demonstrative implementation not intended to have the best performance
+
+// http://encoding.spec.whatwg.org/#textencoder
+
+// http://encoding.spec.whatwg.org/#textencoder
+
+function PTextEncoder() {}
+
+PTextEncoder.prototype.encode = function (string) {
+  const octets = [];
+  const length = string.length;
+  let i = 0;
+  while (i < length) {
+    const codePoint = string.codePointAt(i);
+    let c = 0;
+    let bits = 0;
+    if (codePoint <= 0x0000007f) {
+      c = 0;
+      bits = 0x00;
+    } else if (codePoint <= 0x000007ff) {
+      c = 6;
+      bits = 0xc0;
+    } else if (codePoint <= 0x0000ffff) {
+      c = 12;
+      bits = 0xe0;
+    } else if (codePoint <= 0x001fffff) {
+      c = 18;
+      bits = 0xf0;
+    }
+    octets.push(bits | (codePoint >> c));
+    c -= 6;
+    while (c >= 0) {
+      octets.push(0x80 | ((codePoint >> c) & 0x3f));
+      c -= 6;
+    }
+    i += codePoint >= 0x10000 ? 2 : 1;
+  }
+  return octets;
+};
+
+function PTextDecoder() {}
+
+PTextDecoder.prototype.decode = function (octets) {
+  let string = '';
+  let i = 0;
+  while (i < octets.length) {
+    let octet = octets[i];
+    let bytesNeeded = 0;
+    let codePoint = 0;
+    if (octet <= 0x7f) {
+      bytesNeeded = 0;
+      codePoint = octet & 0xff;
+    } else if (octet <= 0xdf) {
+      bytesNeeded = 1;
+      codePoint = octet & 0x1f;
+    } else if (octet <= 0xef) {
+      bytesNeeded = 2;
+      codePoint = octet & 0x0f;
+    } else if (octet <= 0xf4) {
+      bytesNeeded = 3;
+      codePoint = octet & 0x07;
+    }
+    if (octets.length - i - bytesNeeded > 0) {
+      let k = 0;
+      while (k < bytesNeeded) {
+        octet = octets[i + k + 1];
+        codePoint = (codePoint << 6) | (octet & 0x3f);
+        k += 1;
+      }
+    } else {
+      codePoint = 0xfffd;
+      bytesNeeded = octets.length - i;
+    }
+    string += String.fromCodePoint(codePoint);
+    i += bytesNeeded + 1;
+  }
+  return string;
+};
+
+export const Encoder = (typeof TextEncoder === 'undefined'
+  ? PTextEncoder
+  : TextEncoder) as unknown as typeof TextEncoder;
+export const Decoder = (typeof TextDecoder === 'undefined'
+  ? PTextDecoder
+  : TextDecoder) as unknown as typeof TextDecoder;

packages/core/src/helpers/secret.ts

@@ -0,0 +1,56 @@
+import { Encoder, Decoder } from './encoder';
+
+export const INVISIBLE_CHARACTERS = ['\u200C', '\u200D'];
+
+export const INVISIBLE_REGEX = RegExp(
+  `([${INVISIBLE_CHARACTERS.join('')}]+)`,
+  'gu'
+);
+
+const toBytes = (text: string) => {
+  return Array.from(new Encoder().encode(text));
+};
+
+const fromBytes = (bytes) => {
+  return new Decoder().decode(new Uint8Array(bytes));
+};
+
+const padToWholeBytes = (binary: string) => {
+  const needsToAdd = 8 - binary.length;
+  return '0'.repeat(needsToAdd) + binary;
+};
+
+export const encodeMessage = (text: string) => {
+  const bytes = toBytes(text).map(Number);
+  const binary = bytes
+    .map((byte) => padToWholeBytes(byte.toString(2)) + '0')
+    .join('');
+
+  const result = Array.from(binary)
+    .map((b) => INVISIBLE_CHARACTERS[Number(b)])
+    .join('');
+
+  return result;
+};
+
+const decodeMessage = (message: string) => {
+  const binary = Array.from(message)
+    .map((character) => {
+      return INVISIBLE_CHARACTERS.indexOf(character);
+    })
+    .map(String)
+    .join('');
+
+  const textBytes = binary.match(/(.{9})/g);
+  const codes = Uint8Array.from(
+    textBytes.map((byte) => parseInt(byte.slice(0, 8), 2))
+  );
+  return fromBytes(codes);
+};
+
+export const decodeFromText = (text: string) => {
+  const invisibleMessages = text
+    .match(INVISIBLE_REGEX)
+    ?.filter((m) => m.length > 8);
+  return invisibleMessages?.map(decodeMessage) || [];
+};

packages/core/src/services/DependencyStore.ts

@@ -6,7 +6,7 @@ import { TextService } from './TextService';
 import { MouseEventHandler } from '../highlighter/MouseEventHandler';
 import { TranslationHighlighter } from '../highlighter/TranslationHighlighter';
 import { ElementRegistrar } from './ElementRegistrar';
-import { TextHandler } from '../handlers/TextHandler';
+import { InvisibleTextHandler } from '../handlers/InvisibleTextHandler';
 import { AttributeHandler } from '../handlers/AttributeHandler';
 import { CoreHandler } from '../handlers/CoreHandler';
 import { Observer } from '../Observer';
@@ -17,6 +17,7 @@ import { PluginManager } from '../toolsManager/PluginManager';
 import { Messages } from '../toolsManager/Messages';
 import { HighlightFunctionsInitializer } from '../highlighter/HighlightFunctionsInitializer';
 import { ScreenshotService } from './ScreenshotService';
+import { InvisibleTextService } from './InvisibleTextService';
 
 export class DependencyStore {
   public properties: Properties = new Properties();
@@ -54,10 +55,16 @@ export class DependencyStore {
     this.eventService
   );
 
-  public textHandler = new TextHandler(
+  public invisibleTextService = new InvisibleTextService(
+    this.properties,
+    this.translationService
+  );
+
+  public textHandler = new InvisibleTextHandler(
     this.properties,
     this.translationHighlighter,
     this.textService,
+    this.invisibleTextService,
     this.elementRegistrar
   );
   public attributeHandler = new AttributeHandler(

packages/core/src/services/InvisibleTextService.ts

@@ -0,0 +1,33 @@
+import { KeyAndParams, TranslatedWithMetadata } from '../types';
+import { TranslationService } from './TranslationService';
+import { Properties } from '../Properties';
+import { decodeFromText, INVISIBLE_REGEX } from 'helpers/secret';
+
+export type ReplacedType = { text: string; keys: KeyAndParams[] };
+
+export class InvisibleTextService {
+  constructor(
+    private properties: Properties,
+    private translationService: TranslationService
+  ) {}
+
+  async replace(text: string): Promise<ReplacedType> {
+    const keysAndParams = [] as KeyAndParams[];
+    const keys = decodeFromText(text);
+
+    keys.forEach((key) => {
+      keysAndParams.push({
+        key: key,
+        params: undefined,
+        defaultValue: undefined,
+      });
+    });
+
+    const result = text.replace(INVISIBLE_REGEX, '');
+
+    if (keys.length) {
+      return { text: result, keys: keysAndParams };
+    }
+    return undefined;
+  }
+}

testapps/react-i18next/.gitignore

@@ -0,0 +1,23 @@
+# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
+
+# dependencies
+/node_modules
+/.pnp
+.pnp.js
+
+# testing
+/coverage
+
+# production
+/build
+
+# misc
+.DS_Store
+.env.local
+.env.development.local
+.env.test.local
+.env.production.local
+
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*