diff --git a/fsspec/core.py b/fsspec/core.py index 9643c5cf9..6fe462eda 100644 --- a/fsspec/core.py +++ b/fsspec/core.py @@ -533,7 +533,8 @@ def strip_protocol(urlpath): def expand_paths_if_needed(paths, mode, num, fs, name_function): - """Expand paths if they have a ``*`` in them. + """Expand paths if they have a ``*`` in them (write mode) or any of ``*?[]`` + in them (read mode). :param paths: list of paths mode: str @@ -549,23 +550,32 @@ def expand_paths_if_needed(paths, mode, num, fs, name_function): """ expanded_paths = [] paths = list(paths) - if "w" in mode and sum([1 for p in paths if "*" in p]) > 1: - raise ValueError("When writing data, only one filename mask can be specified.") - elif "w" in mode: + + if "w" in mode: # read mode + if sum([1 for p in paths if "*" in p]) > 1: + raise ValueError( + "When writing data, only one filename mask can be specified." + ) num = max(num, len(paths)) - for curr_path in paths: - if "*" in curr_path: - if "w" in mode: + + for curr_path in paths: + if "*" in curr_path: # expand using name_function expanded_paths.extend(_expand_paths(curr_path, name_function, num)) else: + expanded_paths.append(curr_path) + # if we generated more paths that asked for, trim the list + if len(expanded_paths) > num: + expanded_paths = expanded_paths[:num] + + else: # read mode + for curr_path in paths: + if has_magic(curr_path): # expand using glob expanded_paths.extend(fs.glob(curr_path)) - else: - expanded_paths.append(curr_path) - # if we generated more paths that asked for, trim the list - if "w" in mode and len(expanded_paths) > num: - expanded_paths = expanded_paths[:num] + else: + expanded_paths.append(curr_path) + return expanded_paths diff --git a/fsspec/tests/test_core.py b/fsspec/tests/test_core.py index d4cb6362d..3ccc91b3b 100644 --- a/fsspec/tests/test_core.py +++ b/fsspec/tests/test_core.py @@ -11,6 +11,7 @@ OpenFile, OpenFiles, _expand_paths, + expand_paths_if_needed, get_compression, open_files, open_local, @@ -45,6 +46,30 @@ def test_expand_paths(path, name_function, num, out): assert _expand_paths(path, name_function, num) == out +@pytest.mark.parametrize( + "create_files, path, out", + [ + [["apath"], "apath", ["apath"]], + [["apath1"], "apath*", ["apath1"]], + [["apath1", "apath2"], "apath*", ["apath1", "apath2"]], + [["apath1", "apath2"], "apath[1]", ["apath1"]], + [["apath1", "apath11"], "apath?", ["apath1"]], + ], +) +def test_expand_paths_if_needed_in_read_mode(create_files, path, out): + + d = str(tempfile.mkdtemp()) + for f in create_files: + f = os.path.join(d, f) + open(f, "w").write("test") + + path = os.path.join(d, path) + + fs = fsspec.filesystem("file") + res = expand_paths_if_needed([path], "r", 0, fs, None) + assert [os.path.basename(p) for p in res] == out + + def test_expand_error(): with pytest.raises(ValueError): _expand_paths("*.*", None, 1) @@ -172,8 +197,7 @@ def test_url_kwargs_chain(ftp_writable): f.write(data) with fsspec.open( - f"simplecache::ftp://{username}:{password}@{host}:{port}//afile", - "rb", + f"simplecache::ftp://{username}:{password}@{host}:{port}//afile", "rb" ) as f: assert f.read() == data