Implement String.prototype.matchAll and related (#1731)

Implement `String.prototype.matchAll` and related. This includes:
* `Symbol.matchAll`
* `String.prototype.matchAll`
* `RegExp.prototype[Symbol.matchAll]`

All test262 cases pass; those that do not, fail for pre-existing reasons
unrelated to these changes. Added also some manual tests.

rhino/src/main/java/org/mozilla/javascript/IdScriptableObject.java

@@ -820,18 +820,22 @@ protected void addIdFunctionProperty(
      * @return obj casted to the target type
      * @throws EcmaError if the cast failed.
      */
-    @SuppressWarnings("unchecked")
     protected static <T> T ensureType(Object obj, Class<T> clazz, IdFunctionObject f) {
+        return ensureType(obj, clazz, f.getFunctionName());
+    }
+
+    @SuppressWarnings("unchecked")
+    protected static <T> T ensureType(Object obj, Class<T> clazz, String functionName) {
         if (clazz.isInstance(obj)) {
             return (T) obj;
         }
         if (obj == null) {
             throw ScriptRuntime.typeErrorById(
-                    "msg.incompat.call.details", f.getFunctionName(), "null", clazz.getName());
+                    "msg.incompat.call.details", functionName, "null", clazz.getName());
         }
         throw ScriptRuntime.typeErrorById(
                 "msg.incompat.call.details",
-                f.getFunctionName(),
+                functionName,
                 obj.getClass().getName(),
                 clazz.getName());
     }

rhino/src/main/java/org/mozilla/javascript/LazilyLoadedCtor.java

@@ -35,7 +35,7 @@ public LazilyLoadedCtor(
         this(scope, propertyName, className, sealed, false);
     }
 
-    LazilyLoadedCtor(
+    public LazilyLoadedCtor(
             ScriptableObject scope,
             String propertyName,
             String className,

rhino/src/main/java/org/mozilla/javascript/NativeString.java

@@ -240,6 +240,10 @@ protected void initPrototypeId(int id) {
                 arity = 1;
                 s = "match";
                 break;
+            case Id_matchAll:
+                arity = 1;
+                s = "matchAll";
+                break;
             case Id_search:
                 arity = 1;
                 s = "search";
@@ -859,6 +863,60 @@ else if (Normalizer.Form.NFC.name().equals(formStr))
 
                 case SymbolId_iterator:
                     return new NativeStringIterator(scope, requireObjectCoercible(cx, thisObj, f));
+
+                case Id_matchAll:
+                    {
+                        // See ECMAScript spec 22.1.3.14
+                        Object o = requireObjectCoercible(cx, thisObj, f);
+                        Object regexp = args.length > 0 ? args[0] : Undefined.instance;
+                        RegExpProxy regExpProxy = ScriptRuntime.checkRegExpProxy(cx);
+                        if (regexp != null && !Undefined.isUndefined(regexp)) {
+                            boolean isRegExp =
+                                    regexp instanceof Scriptable
+                                            && regExpProxy.isRegExp((Scriptable) regexp);
+                            if (isRegExp) {
+                                Object flags =
+                                        ScriptRuntime.getObjectProp(regexp, "flags", cx, scope);
+                                requireObjectCoercible(cx, flags, f);
+                                String flagsStr = ScriptRuntime.toString(flags);
+                                if (!flagsStr.contains("g")) {
+                                    throw ScriptRuntime.typeErrorById(
+                                            "msg.str.match.all.no.global.flag");
+                                }
+                            }
+
+                            Object matcher =
+                                    ScriptRuntime.getObjectElem(
+                                            regexp, SymbolKey.MATCH_ALL, cx, scope);
+                            // If method is not undefined, it should be a Callable
+                            if (matcher != null && !Undefined.isUndefined(matcher)) {
+                                if (!(matcher instanceof Callable)) {
+                                    throw ScriptRuntime.notFunctionError(
+                                            regexp, matcher, SymbolKey.MATCH_ALL.getName());
+                                }
+                                return ((Callable) matcher)
+                                        .call(
+                                                cx,
+                                                scope,
+                                                ScriptRuntime.toObject(scope, regexp),
+                                                new Object[] {o});
+                            }
+                        }
+
+                        String s = ScriptRuntime.toString(o);
+                        String regexpToString =
+                                Undefined.isUndefined(regexp) ? "" : ScriptRuntime.toString(regexp);
+                        Object compiledRegExp = regExpProxy.compileRegExp(cx, regexpToString, "g");
+                        Scriptable rx = regExpProxy.wrapRegExp(cx, scope, compiledRegExp);
+
+                        Object method =
+                                ScriptRuntime.getObjectElem(rx, SymbolKey.MATCH_ALL, cx, scope);
+                        if (!(method instanceof Callable)) {
+                            throw ScriptRuntime.notFunctionError(
+                                    rx, method, SymbolKey.MATCH_ALL.getName());
+                        }
+                        return ((Callable) method).call(cx, scope, rx, new Object[] {s});
+                    }
             }
             throw new IllegalArgumentException(
                     "String.prototype has no method: " + f.getFunctionName());
@@ -1382,6 +1440,9 @@ protected int findPrototypeId(String s) {
             case "match":
                 id = Id_match;
                 break;
+            case "matchAll":
+                id = Id_matchAll;
+                break;
             case "search":
                 id = Id_search;
                 break;
@@ -1512,7 +1573,8 @@ protected int findPrototypeId(String s) {
             Id_at = 52,
             Id_isWellFormed = 53,
             Id_toWellFormed = 54,
-            MAX_PROTOTYPE_ID = Id_toWellFormed;
+            Id_matchAll = 55,
+            MAX_PROTOTYPE_ID = Id_matchAll;
     private static final int ConstructorId_charAt = -Id_charAt,
             ConstructorId_charCodeAt = -Id_charCodeAt,
             ConstructorId_indexOf = -Id_indexOf,

rhino/src/main/java/org/mozilla/javascript/NativeSymbol.java

@@ -72,6 +72,7 @@ public static void init(Context cx, Scriptable scope, boolean sealed) {
         createStandardSymbol(cx, scope, ctor, "isRegExp", SymbolKey.IS_REGEXP);
         createStandardSymbol(cx, scope, ctor, "toPrimitive", SymbolKey.TO_PRIMITIVE);
         createStandardSymbol(cx, scope, ctor, "match", SymbolKey.MATCH);
+        createStandardSymbol(cx, scope, ctor, "matchAll", SymbolKey.MATCH_ALL);
         createStandardSymbol(cx, scope, ctor, "replace", SymbolKey.REPLACE);
         createStandardSymbol(cx, scope, ctor, "search", SymbolKey.SEARCH);
         createStandardSymbol(cx, scope, ctor, "split", SymbolKey.SPLIT);

rhino/src/main/java/org/mozilla/javascript/RegExpProxy.java

@@ -18,6 +18,8 @@ public interface RegExpProxy {
     public static final int RA_REPLACE_ALL = 3;
     public static final int RA_SEARCH = 4;
 
+    public void register(ScriptableObject scope, boolean sealed);
+
     public boolean isRegExp(Scriptable obj);
 
     public Object compileRegExp(Context cx, String source, String flags);

rhino/src/main/java/org/mozilla/javascript/ScriptRuntime.java

@@ -188,6 +188,7 @@ public static ScriptableObject initSafeStandardObjects(
 
         NativeArrayIterator.init(scope, sealed);
         NativeStringIterator.init(scope, sealed);
+        registerRegExp(cx, scope, sealed);
 
         NativeJavaObject.init(scope, sealed);
         NativeJavaMap.init(scope, sealed);
@@ -196,8 +197,6 @@ public static ScriptableObject initSafeStandardObjects(
                 cx.hasFeature(Context.FEATURE_E4X) && cx.getE4xImplementationFactory() != null;
 
         // define lazy-loaded properties using their class name
-        new LazilyLoadedCtor(
-                scope, "RegExp", "org.mozilla.javascript.regexp.NativeRegExp", sealed, true);
         new LazilyLoadedCtor(
                 scope, "Continuation", "org.mozilla.javascript.NativeContinuation", sealed, true);
 
@@ -299,6 +298,13 @@ public static ScriptableObject initSafeStandardObjects(
         return scope;
     }
 
+    private static void registerRegExp(Context cx, ScriptableObject scope, boolean sealed) {
+        RegExpProxy regExpProxy = getRegExpProxy(cx);
+        if (regExpProxy != null) {
+            regExpProxy.register(scope, sealed);
+        }
+    }
+
     public static ScriptableObject initStandardObjects(
             Context cx, ScriptableObject scope, boolean sealed) {
         ScriptableObject s = initSafeStandardObjects(cx, scope, sealed);
@@ -1388,6 +1394,14 @@ public static long toLength(Object[] args, int index) {
         return (long) Math.min(len, NativeNumber.MAX_SAFE_INTEGER);
     }
 
+    public static long toLength(Object value) {
+        double len = toInteger(value);
+        if (len <= 0.0) {
+            return 0;
+        }
+        return (long) Math.min(len, NativeNumber.MAX_SAFE_INTEGER);
+    }
+
     /** See ECMA 9.5. */
     public static int toInt32(Object val) {
         // short circuit for common integer values
@@ -1444,6 +1458,20 @@ public static Optional<Double> canonicalNumericIndexString(String arg) {
         return Optional.empty();
     }
 
+    /** Implements the abstract operation AdvanceStringIndex. See ECMAScript spec 22.2.7.3 */
+    public static long advanceStringIndex(String string, long index, boolean unicode) {
+        if (index >= NativeNumber.MAX_SAFE_INTEGER) Kit.codeBug();
+        if (!unicode) {
+            return index + 1;
+        }
+        int length = string.length();
+        if (index + 1 > length) {
+            return index + 1;
+        }
+        int cp = string.codePointAt((int) index);
+        return index + Character.charCount(cp);
+    }
+
     // XXX: this is until setDefaultNamespace will learn how to store NS
     // properly and separates namespace form Scriptable.get etc.
     private static final String DEFAULT_NS_TAG = "__default_namespace__";

rhino/src/main/java/org/mozilla/javascript/SymbolKey.java

@@ -20,6 +20,7 @@ public class SymbolKey implements Symbol, Serializable {
     public static final SymbolKey IS_REGEXP = new SymbolKey("Symbol.isRegExp");
     public static final SymbolKey TO_PRIMITIVE = new SymbolKey("Symbol.toPrimitive");
     public static final SymbolKey MATCH = new SymbolKey("Symbol.match");
+    public static final SymbolKey MATCH_ALL = new SymbolKey("Symbol.matchAll");
     public static final SymbolKey REPLACE = new SymbolKey("Symbol.replace");
     public static final SymbolKey SEARCH = new SymbolKey("Symbol.search");
     public static final SymbolKey SPLIT = new SymbolKey("Symbol.split");

rhino/src/main/java/org/mozilla/javascript/regexp/NativeRegExp.java

@@ -7,7 +7,10 @@
 package org.mozilla.javascript.regexp;
 
 import java.io.Serializable;
+import org.mozilla.javascript.AbstractEcmaObjectOperations;
+import org.mozilla.javascript.Constructable;
 import org.mozilla.javascript.Context;
+import org.mozilla.javascript.Function;
 import org.mozilla.javascript.IdFunctionObject;
 import org.mozilla.javascript.IdScriptableObject;
 import org.mozilla.javascript.Kit;
@@ -2695,6 +2698,10 @@ protected void initPrototypeId(int id) {
             initPrototypeMethod(REGEXP_TAG, id, SymbolKey.MATCH, "[Symbol.match]", 1);
             return;
         }
+        if (id == SymbolId_matchAll) {
+            initPrototypeMethod(REGEXP_TAG, id, SymbolKey.MATCH_ALL, "[Symbol.matchAll]", 1);
+            return;
+        }
         if (id == SymbolId_search) {
             initPrototypeMethod(REGEXP_TAG, id, SymbolKey.SEARCH, "[Symbol.search]", 1);
             return;
@@ -2761,7 +2768,7 @@ public Object execIdCall(
                 return realThis(thisObj, f).toString();
 
             case Id_exec:
-                return realThis(thisObj, f).execSub(cx, scope, args, MATCH);
+                return js_exec(cx, scope, thisObj, args);
 
             case Id_test:
                 {
@@ -2775,6 +2782,9 @@ public Object execIdCall(
             case SymbolId_match:
                 return realThis(thisObj, f).execSub(cx, scope, args, MATCH);
 
+            case SymbolId_matchAll:
+                return js_SymbolMatchAll(cx, scope, thisObj, args);
+
             case SymbolId_search:
                 Scriptable scriptable =
                         (Scriptable) realThis(thisObj, f).execSub(cx, scope, args, MATCH);
@@ -2783,15 +2793,54 @@ public Object execIdCall(
         throw new IllegalArgumentException(String.valueOf(id));
     }
 
+    static Object js_exec(Context cx, Scriptable scope, Scriptable thisObj, Object[] args) {
+        return realThis(thisObj, "exec").execSub(cx, scope, args, MATCH);
+    }
+
+    private Object js_SymbolMatchAll(
+            Context cx, Scriptable scope, Scriptable thisObj, Object[] args) {
+        // See ECMAScript spec 22.2.6.9
+        if (!ScriptRuntime.isObject(thisObj)) {
+            throw ScriptRuntime.typeErrorById("msg.arg.not.object", ScriptRuntime.typeof(thisObj));
+        }
+
+        String s = ScriptRuntime.toString(args.length > 0 ? args[0] : Undefined.instance);
+
+        Scriptable topLevelScope = ScriptableObject.getTopLevelScope(scope);
+        Function defaultConstructor =
+                ScriptRuntime.getExistingCtor(cx, topLevelScope, getClassName());
+        Constructable c =
+                AbstractEcmaObjectOperations.speciesConstructor(cx, thisObj, defaultConstructor);
+
+        String flags = ScriptRuntime.toString(ScriptRuntime.getObjectProp(thisObj, "flags", cx));
+
+        Scriptable matcher = c.construct(cx, scope, new Object[] {thisObj, flags});
+
+        long lastIndex =
+                ScriptRuntime.toLength(ScriptRuntime.getObjectProp(thisObj, "lastIndex", cx));
+        ScriptRuntime.setObjectProp(matcher, "lastIndex", lastIndex, cx);
+        boolean global = flags.indexOf('g') != -1;
+        boolean fullUnicode = flags.indexOf('u') != -1 || flags.indexOf('v') != -1;
+
+        return new NativeRegExpStringIterator(scope, matcher, s, global, fullUnicode);
+    }
+
     private static NativeRegExp realThis(Scriptable thisObj, IdFunctionObject f) {
-        return ensureType(thisObj, NativeRegExp.class, f);
+        return realThis(thisObj, f.getFunctionName());
+    }
+
+    private static NativeRegExp realThis(Scriptable thisObj, String functionName) {
+        return ensureType(thisObj, NativeRegExp.class, functionName);
     }
 
     @Override
     protected int findPrototypeId(Symbol k) {
         if (SymbolKey.MATCH.equals(k)) {
             return SymbolId_match;
         }
+        if (SymbolKey.MATCH_ALL.equals(k)) {
+            return SymbolId_matchAll;
+        }
         if (SymbolKey.SEARCH.equals(k)) {
             return SymbolId_search;
         }
@@ -2834,7 +2883,8 @@ protected int findPrototypeId(String s) {
             Id_test = 5,
             Id_prefix = 6,
             SymbolId_match = 7,
-            SymbolId_search = 8,
+            SymbolId_matchAll = 8,
+            SymbolId_search = 9,
             MAX_PROTOTYPE_ID = SymbolId_search;
 
     private RECompiled re;