Support wildcard include using glob syntax

HOCON.md

@@ -896,9 +896,10 @@ An _include statement_ consists of the unquoted string `include`
 followed by whitespace and then either:
  - a single _quoted_ string which is interpreted heuristically as
    URL, filename, or classpath resource.
- - `url()`, `file()`, or `classpath()` surrounding a quoted string
-   which is then interpreted as a URL, file, or classpath. The
-   string must be quoted, unlike in CSS.
+ - `url()`, `file()`, `glob()`, or `classpath()` surrounding a
+   quoted string which is then interpreted as a URL, file, file
+   wildcard/glob pattern, or classpath. The string must be quoted,
+   unlike in CSS.
  - `required()` surrounding one of the above
 
 An include statement can appear in place of an object field.
@@ -908,7 +909,7 @@ expression where an object key would be expected, then it is not
 interpreted as a path expression or a key.
 
 Instead, the next value must be a _quoted_ string or a quoted
-string surrounded by `url()`, `file()`, or `classpath()`.
+string surrounded by `url()`, `file()`, `glob()`, or `classpath()`.
 This value is the _resource name_.
 
 Together, the unquoted `include` and the resource name substitute
@@ -918,8 +919,8 @@ usual the comma may be omitted if there's a newline).
 
 If an unquoted `include` at the start of a key is followed by
 anything other than a single quoted string or the
-`url("")`/`file("")`/`classpath("")` syntax, it is invalid and an
-error should be generated.
+`url("")`/`file("")`/`glob("")`/`classpath("")` syntax, it is
+invalid and an error should be generated.
 
 There can be any amount of whitespace, including newlines, between
 the unquoted `include` and the resource name. For `url()` etc.,
@@ -1080,9 +1081,9 @@ for file: URLs.
 
 #### Include semantics: locating resources
 
-A quoted string not surrounded by `url()`, `file()`, `classpath()`
-must be interpreted heuristically. The heuristic is to treat the
-quoted string as:
+A quoted string not surrounded by `url()`, `file()`, `glob()`,
+`classpath()` must be interpreted heuristically. The heuristic is
+to treat the quoted string as:
 
  - a URL, if the quoted string is a valid URL with a known
    protocol.
@@ -1150,8 +1151,9 @@ they do support them they should do so as described above.
 
 Note that at present, if `url()`/`file()`/`classpath()` are
 specified, the included items are NOT interpreted relative to the
-including items. Relative-to-including-file paths only work with
-the heuristic `include "foo.conf"`. This may change in the future.
+including items. Relative-to-including-file paths work with the
+heuristic `include "foo.conf"` and with `glob()` (only if it's
+used from the file). This may change in the future.
 
 ### Conversion of numerically-indexed objects to arrays
 

README.md

@@ -524,6 +524,7 @@ tree that you could have written (less conveniently) in JSON.
     "whatever"` to have the library do what you probably mean
     (Note: `url()`/`file()`/`classpath()` syntax is not supported
     in Play/Akka 2.0, only in later releases.)
+  - you can include files by wildcard/glob pattern; use `glob()`.
   - substitutions `foo : ${a.b}` sets key `foo` to the same value
     as the `b` field in the `a` object
   - substitutions concatenate into unquoted strings, `foo : the

config/src/main/java/com/typesafe/config/ConfigIncluder.java

@@ -8,7 +8,7 @@
  * {@link ConfigParseOptions#setIncluder ConfigParseOptions.setIncluder()} to
  * customize handling of {@code include} statements in config files. You may
  * also want to implement {@link ConfigIncluderClasspath},
- * {@link ConfigIncluderFile}, and {@link ConfigIncluderURL}, or not.
+ * {@link ConfigIncluderFile}, {@link ConfigIncluderGlob}, and {@link ConfigIncluderURL}, or not.
  */
 public interface ConfigIncluder {
     /**

config/src/main/java/com/typesafe/config/ConfigIncluderGlob.java

@@ -0,0 +1,25 @@
+/**
+ *   Copyright (C) 2011-2012 Typesafe Inc. <http://typesafe.com>
+ */
+package com.typesafe.config;
+
+/**
+ * Implement this <em>in addition to</em> {@link ConfigIncluder} if you want to
+ * support inclusion of files by wildcard with the {@code include glob("wilcard")} syntax.
+ * If you do not implement this but do implement {@link ConfigIncluder},
+ * attempts to load files by wilcard will use the default includer.
+ */
+public interface ConfigIncluderGlob {
+    /**
+     * Parses another item to be included. The returned object typically would
+     * not have substitutions resolved. You can throw a ConfigException here to
+     * abort parsing, or return an empty object, but may not return null.
+     *
+     * @param context
+     *            some info about the include context
+     * @param what
+     *            the include statement's argument
+     * @return a non-null ConfigObject
+     */
+    ConfigObject includeGlob(ConfigIncludeContext context, String what);
+}

config/src/main/java/com/typesafe/config/impl/ConfigDocumentParser.java

@@ -366,6 +366,9 @@ private ConfigNodeInclude parseIncludeResource(ArrayList<AbstractConfigNode> chi
                 if (kindText.startsWith("url(")) {
                     kind = ConfigIncludeKind.URL;
                     prefix = "url(";
+                } else if (kindText.startsWith("glob(")) {
+                    kind = ConfigIncludeKind.GLOB;
+                    prefix = "glob(";
                 } else if (kindText.startsWith("file(")) {
                     kind = ConfigIncludeKind.FILE;
                     prefix = "file(";

config/src/main/java/com/typesafe/config/impl/ConfigIncludeKind.java

@@ -1,5 +1,5 @@
 package com.typesafe.config.impl;
 
 enum ConfigIncludeKind {
-    URL, FILE, CLASSPATH, HEURISTIC
+    URL, GLOB, FILE, CLASSPATH, HEURISTIC
 }

config/src/main/java/com/typesafe/config/impl/ConfigParser.java

@@ -172,6 +172,10 @@ private void parseInclude(Map<String, AbstractConfigValue> values, ConfigNodeInc
                     obj = (AbstractConfigObject) includer.includeURL(cic, url);
                     break;
 
+                case GLOB:
+                    obj = (AbstractConfigObject) includer.includeGlob(cic, n.name());
+                    break;
+
                 case FILE:
                     obj = (AbstractConfigObject) includer.includeFile(cic,
                             new File(n.name()));

config/src/main/java/com/typesafe/config/impl/FullIncluder.java

@@ -6,9 +6,10 @@
 import com.typesafe.config.ConfigIncluder;
 import com.typesafe.config.ConfigIncluderClasspath;
 import com.typesafe.config.ConfigIncluderFile;
+import com.typesafe.config.ConfigIncluderGlob;
 import com.typesafe.config.ConfigIncluderURL;
 
 interface FullIncluder extends ConfigIncluder, ConfigIncluderFile, ConfigIncluderURL,
-            ConfigIncluderClasspath {
+            ConfigIncluderClasspath, ConfigIncluderGlob {
 
 }

config/src/main/java/com/typesafe/config/impl/Parseable.java

@@ -19,7 +19,13 @@
 import java.net.URISyntaxException;
 import java.net.URL;
 import java.net.URLConnection;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.PathMatcher;
+import java.nio.file.Paths;
 import java.util.*;
+import java.util.function.UnaryOperator;
+import java.util.regex.PatternSyntaxException;
 
 import com.typesafe.config.*;
 import com.typesafe.config.parser.*;
@@ -54,6 +60,8 @@ protected LinkedList<Parseable> initialValue() {
 
     private static final int MAX_INCLUDE_DEPTH = 50;
 
+    private static final int MAX_FILES_SEARCH_DEPTH = 10;
+
     protected Parseable() {
     }
 
@@ -670,6 +678,129 @@ public static Parseable newFile(File input, ConfigParseOptions options) {
         return new ParseableFile(input, options);
     }
 
+    private final static class ParseableGlob extends Parseable {
+        final private String pattern;
+        final private Path relativeTo;
+
+        ParseableGlob(String pattern, Path relativeTo, ConfigParseOptions options) {
+            this.pattern = pattern;
+            relativeTo = relativeTo.toAbsolutePath();
+            if (!relativeTo.toFile().isDirectory())
+                relativeTo = relativeTo.getParent();
+            this.relativeTo = relativeTo;
+            postConstruct(options);
+        }
+
+        @Override
+        protected Reader reader() throws IOException {
+            throw new ConfigException.BugOrBroken("reader() should not be called on glob");
+        }
+
+        @Override
+        protected AbstractConfigObject rawParseValue(ConfigOrigin origin, ConfigParseOptions finalOptions)
+                throws IOException {
+            Path searchpath = globSearchBasePath(pattern);
+            UnaryOperator<Path> relativize = searchpath != null && searchpath.isAbsolute() ?
+                                                 UnaryOperator.identity() : relativeTo::relativize;
+            if (searchpath == null)
+                searchpath = relativeTo;
+            else if (!searchpath.isAbsolute())
+                searchpath = relativeTo.resolve(searchpath);
+
+            PathMatcher matcher = getMatcher(searchpath);
+            Iterable<AbstractConfigValue> objects = Files.find(searchpath, MAX_FILES_SEARCH_DEPTH, (p, fa) -> matcher.matches(relativize.apply(p)))
+                    .map(this::newFile).map(Parseable::parseValue)::iterator;
+
+            AbstractConfigObject merged = null;
+            for (AbstractConfigValue v : objects) {
+                if (merged == null) {
+                    merged = SimpleConfigObject.empty(origin);
+                }
+                merged = merged.withFallback(v);
+            }
+            if (merged == null)
+                throw new IOException("No files found by glob pattern '" + pattern + "' relative to path '" + relativeTo + "'");
+
+            return merged;
+        }
+
+        private PathMatcher getMatcher(Path path) {
+            try {
+                return path.getFileSystem().getPathMatcher("glob:" + pattern);
+            } catch (PatternSyntaxException e) {
+                // may be should return including file origin?
+                throw new ConfigException.IO(origin(), "Invalid glob pattern: " + e.getMessage(), e);
+            }
+        }
+
+        private Parseable newFile(Path path) {
+            File file = path.toFile();
+            if (ConfigImpl.traceLoadsEnabled())
+                trace("Loading config from file '" + file + "' found by 'glob:" + pattern + "'");
+            if (!file.canRead())
+                throw new ConfigException.IO(origin(), "File '" + path + "' found by glob pattern is not readable");
+            return Parseable.newFile(file, options());
+        }
+
+        @Override
+        ConfigSyntax guessSyntax() {
+            return ConfigImplUtil.syntaxFromExtension(pattern);
+        }
+
+        @Override
+        protected ConfigOrigin createOrigin() {
+            return SimpleConfigOrigin.newSimple(pattern);
+        }
+
+        @Override
+        public String toString() {
+            return getClass().getSimpleName() + "(glob:" + pattern + ")";
+        }
+    }
+
+    static Path globSearchBasePath(String pattern) {
+        // searching for the last slash before first wildcard character (the constant subpath of pattern)
+        int noWildcardSlash = 0;
+        StringBuilder path = new StringBuilder(pattern.length());
+        loop:
+        for (int i = 0; i < pattern.length(); ++i) {
+            char ch = pattern.charAt(i);
+            switch(ch) {
+            case '*':
+            case '?':
+            case '[':
+            case '{':
+                break loop;
+            case '\\':
+                if (++i < pattern.length() && ".[{(^$".indexOf(ch = pattern.charAt(i)) != -1) {
+                    // glob or regex metacharacter valid in path
+                    path.append(ch);
+                    break;
+                }
+                break loop;
+            case '/':
+                noWildcardSlash = path.length() + 1;
+                // no break
+            default:
+                path.append(ch);
+            }
+        }
+        if (noWildcardSlash == 0)
+            return null;
+        return Paths.get(path.substring(0, noWildcardSlash));
+    }
+
+    public static ConfigParseable newGlob(String pattern, ConfigParseOptions options) {
+        LinkedList<Parseable> stack = parseStack.get();
+        if (stack.isEmpty())
+            throw new ConfigException.BugOrBroken("newGlob(String, ConfigParseOptions) called with empty parseStack (including glob outside of parse process?)");
+
+        Path relativePath = Paths.get("");
+        // may be should walk through stack to find the first ParseableFile item?
+        if (stack.getFirst() instanceof ParseableFile)
+            relativePath = ((ParseableFile) stack.getFirst()).input.toPath();
+        return new ParseableGlob(pattern, relativePath, options);
+    }
 
     private final static class ParseableResourceURL extends ParseableURL {
 

config/src/main/java/com/typesafe/config/impl/SimpleIncluder.java

@@ -15,6 +15,7 @@
 import com.typesafe.config.ConfigIncluder;
 import com.typesafe.config.ConfigIncluderClasspath;
 import com.typesafe.config.ConfigIncluderFile;
+import com.typesafe.config.ConfigIncluderGlob;
 import com.typesafe.config.ConfigIncluderURL;
 import com.typesafe.config.ConfigObject;
 import com.typesafe.config.ConfigParseOptions;
@@ -123,6 +124,26 @@ static ConfigObject includeResourceWithoutFallback(final ConfigIncludeContext co
         return ConfigFactory.parseResourcesAnySyntax(resource, context.parseOptions()).root();
     }
 
+    @Override
+    public ConfigObject includeGlob(ConfigIncludeContext context, String pattern) {
+        ConfigObject obj = includeGlobWithoutFallback(context, pattern);
+
+        // now use the fallback includer if any and merge
+        // its result.
+        if (fallback != null && fallback instanceof ConfigIncluderGlob) {
+            return obj.withFallback(((ConfigIncluderGlob) fallback).includeGlob(context,
+                    pattern));
+        } else {
+            return obj;
+        }
+    }
+
+    static ConfigObject includeGlobWithoutFallback(final ConfigIncludeContext context,
+            String pattern) {
+        ConfigParseable parseable = Parseable.newGlob(pattern, context.parseOptions());
+        return parseable.parse(parseable.options());
+    }
+
     @Override
     public ConfigIncluder withFallback(ConfigIncluder fallback) {
         if (this == fallback) {
@@ -291,6 +312,14 @@ public ConfigObject includeFile(ConfigIncludeContext context, File what) {
             else
                 return includeFileWithoutFallback(context, what);
         }
+
+        @Override
+        public ConfigObject includeGlob(ConfigIncludeContext context, String what) {
+            if (delegate instanceof ConfigIncluderGlob)
+                return ((ConfigIncluderGlob) delegate).includeGlob(context, what);
+            else
+                return includeGlobWithoutFallback(context, what);
+        }
     }
 
     static FullIncluder makeFull(ConfigIncluder includer) {

config/src/test/resources/file-include-glob.conf

@@ -0,0 +1 @@
+include "subdir/glob.conf"

config/src/test/resources/subdir/a.conf

@@ -0,0 +1 @@
+theconf = 123

config/src/test/resources/subdir/glob.conf

@@ -0,0 +1,3 @@
+glob {
+    include glob("subdir/a.{conf,json,properties}")
+}

config/src/test/resources/subdir/subdir/a.conf

@@ -0,0 +1 @@
+conf = 1

config/src/test/resources/subdir/subdir/a.json

@@ -0,0 +1,3 @@
+{
+    "json": 2
+}

config/src/test/resources/subdir/subdir/a.properties

@@ -0,0 +1 @@
+properties=3

config/src/test/scala/com/typesafe/config/impl/ParseableSearchBasePath.scala

@@ -0,0 +1,27 @@
+package com.typesafe.config.impl
+
+import java.nio.file.Paths;
+
+import org.junit.Assert.{ assertEquals, assertNull }
+import org.junit.Test
+
+class ParseableSearchBasePath extends TestUtils {
+
+    @Test
+    def testGlobSearchBasePath(): Unit = {
+        assertEquals(Paths.get("/a/b/c"), Parseable.globSearchBasePath("/a/b/c/d.{conf,json,properties}"))
+        assertEquals(Paths.get("/a/b"), Parseable.globSearchBasePath("/a/b/c?/d.{conf,json,properties}"))
+        assertEquals(Paths.get("/a/b"), Parseable.globSearchBasePath("/a/b/c[fx]/d.{conf,json,properties}"))
+        assertEquals(Paths.get("/a/b/c"), Parseable.globSearchBasePath("/a/b/c/*.conf"))
+        assertEquals(Paths.get("/"), Parseable.globSearchBasePath("/*.conf"))
+        assertEquals(Paths.get("/a/b/c[$]{^}"), Parseable.globSearchBasePath("/a/b/c\\[\\$]\\{\\^}/*.conf"))
+
+        assertNull(Parseable.globSearchBasePath("*.conf"))
+        assertNull(Parseable.globSearchBasePath("abc"))
+
+        assertEquals(Paths.get("abc"), Parseable.globSearchBasePath("abc/*.conf"))
+        assertEquals(Paths.get("a/b"), Parseable.globSearchBasePath("a/b/c?/d.conf"))
+        assertEquals(Paths.get("a"), Parseable.globSearchBasePath("a/b[cd]/*.conf"))
+        assertEquals(Paths.get("a"), Parseable.globSearchBasePath("a/b.conf"))
+    }
+}