diff --git a/physical/file/file.go b/physical/file/file.go
index b02efc78c22e..f7f51928af61 100644
--- a/physical/file/file.go
+++ b/physical/file/file.go
@@ -247,10 +247,16 @@ func (b *FileBackend) ListInternal(prefix string) ([]string, error) {
 	}
 
 	for i, name := range names {
-		if name[0] == '_' {
-			names[i] = name[1:]
-		} else {
+		fi, err := os.Stat(filepath.Join(path, name))
+		if err != nil {
+			return nil, err
+		}
+		if fi.IsDir() {
 			names[i] = name + "/"
+		} else {
+			if name[0] == '_' {
+				names[i] = name[1:]
+			}
 		}
 	}
 
diff --git a/physical/testing.go b/physical/testing.go
index 4c7537860877..6826a95acc64 100644
--- a/physical/testing.go
+++ b/physical/testing.go
@@ -194,6 +194,45 @@ func ExerciseBackend(t *testing.T, b Backend) {
 	if len(keys) != 0 {
 		t.Fatalf("bad: %v", keys)
 	}
+
+	// Underscores should not trip things up; ref GH-3476
+	e = &Entry{Key: "_zip/_zap", Value: []byte("foobar")}
+	err = b.Put(e)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	e, err = b.Get("_zip/_zap")
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	if e == nil {
+		t.Fatal("got nil entry")
+	}
+	vals, err := b.List("")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(vals) != 1 || vals[0] != "_zip/" {
+		t.Fatalf("bad: %v", vals)
+	}
+	vals, err = b.List("_zip/")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(vals) != 1 || vals[0] != "_zap" {
+		t.Fatalf("bad: %v", vals)
+	}
+	err = b.Delete("_zip/_zap")
+	if err != nil {
+		t.Fatal(err)
+	}
+	vals, err = b.List("")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(vals) != 0 {
+		t.Fatalf("bad: %v", vals)
+	}
 }
 
 func ExerciseBackend_ListPrefix(t *testing.T, b Backend) {