add-go-fuse-to-inject-filesystem-error

.travis.yml

@@ -1,20 +1,37 @@
 dist: trusty
 language: go
-os:
-  - windows
-  - linux
-  - osx
-
-go:
-  - 1.12.x
-
 go_import_path: github.com/prometheus/tsdb
+matrix:
+  include:
+  - os: linux
+    dist: trusty
+    sudo: required
+    go: 1.12.x
+    install:
+      - make deps
+    script:
+      - sudo -E apt-get -yq --no-install-suggests --no-install-recommends --force-yes install fuse
+      - sudo modprobe fuse
+      - sudo chmod a+rw /etc/fuse.conf
+      - sudo echo user_allow_other >> /etc/fuse.conf
+      - make all
+  - os: osx
+    go: 1.12.x
+    sudo: required
+    osx_image: xcode8.3
+    script:
+      - brew update
+      - brew tap homebrew/cask
+      - brew cask install osxfuse
+      - sudo /Library/Filesystems/osxfuse.fs/Contents/Resources/load_osxfuse
+      - sudo sysctl -w vfs.generic.osxfuse.tunables.allow_other=1
+      - make all
+  - os: windows
+    go: 1.12.x
+    before_install:
+      - choco install make
+    install:
+      - make deps
+    script:
+      - make test
 
-before_install:
-  - if [[ "$TRAVIS_OS_NAME" == "windows" ]]; then choco install make; fi
-
-install:
-  - make deps
-
-script:
-  - if [[ "$TRAVIS_OS_NAME" == "windows" ]]; then make test; else make all; fi

compact_test.go

@@ -16,6 +16,8 @@ package tsdb
 import (
 	"context"
 	"fmt"
+	"github.com/go-kit/kit/log/level"
+	"github.com/prometheus/tsdb/testutil/fuse"
 	"io/ioutil"
 	"math"
 	"os"
@@ -1058,3 +1060,38 @@ func TestDeleteCompactionBlockAfterFailedReload(t *testing.T) {
 		})
 	}
 }
+
+// TestOpenBlockWithHook ensures that when OpenBlock call rename failed, there is not meta.json file
+// It use fuse to inject rename filesystem error
+func TestOpenBlockWithHook(t *testing.T) {
+	// Init action.
+	original, err := ioutil.TempDir("", "dev")
+	testutil.Ok(t, err)
+	mountpoint, err := ioutil.TempDir("", "mountpoint")
+	testutil.Ok(t, err)
+
+	// Create block will be successful because hook server does not start.
+	_, file := filepath.Split(createBlock(t, original, genSeries(1, 1, 200, 300)))
+	clean := fuse.NewServer(t, original, mountpoint, fuse.Hook(fuse.TestRenameHook{}))
+	// Remember to call unmount after you do not use it.
+	defer clean()
+
+	// Normal logic.
+	pb, err := OpenBlock(nil, filepath.Join(mountpoint, file), nil)
+	if err != nil {
+		level.Warn(log.NewNopLogger()).Log("msg", "couldn't write the meta file for the block size", "err", err)
+		return
+	}
+	testutil.Ok(t, pb.chunkr.Close())
+	testutil.Ok(t, pb.indexr.Close())
+	testutil.Ok(t, pb.tombstones.Close())
+
+	testutil.Equals(t, true, Exist(filepath.Join(mountpoint, file, "index")))
+	// Rename failed because there is an error by injected file system exception with fuse.
+	testutil.Equals(t, false, Exist(filepath.Join(mountpoint, file, "meta.json")))
+}
+
+func Exist(filename string) bool {
+	_, err := os.Stat(filename)
+	return err == nil || os.IsExist(err)
+}

go.mod

@@ -4,10 +4,11 @@ require (
 	github.com/cespare/xxhash v1.1.0
 	github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954
 	github.com/go-kit/kit v0.8.0
+	github.com/hanwen/go-fuse v1.0.0
 	github.com/oklog/ulid v1.3.1
 	github.com/pkg/errors v0.8.0
 	github.com/prometheus/client_golang v1.0.0
 	golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4
-	golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5
+	golang.org/x/sys v0.0.0-20190422165155-953cdadca894
 	gopkg.in/alecthomas/kingpin.v2 v2.2.6
 )

go.sum

@@ -27,6 +27,8 @@ github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/hanwen/go-fuse v1.0.0 h1:GxS9Zrn6c35/BnfiVsZVWmsG803xwE7eVRDvcf/BEVc=
+github.com/hanwen/go-fuse v1.0.0/go.mod h1:unqXarDXqzAk0rt98O2tVndEPIpUgLD9+rwFisZH3Ok=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
@@ -72,9 +74,12 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f h1:Bl/8QSvNqXvPGPGXa2z5xUTm
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5 h1:mzjBh+S5frKOsOBobWIMAbXavqjmgO17k/2puhcFR94=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

testutil/fuse/fuse.go

@@ -0,0 +1,130 @@
+// +build linux darwin
+
+// Copyright 2019 The qiffang Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package fuse
+
+import (
+	"fmt"
+	"github.com/prometheus/tsdb/testutil"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/hanwen/go-fuse/fuse"
+	"github.com/hanwen/go-fuse/fuse/nodefs"
+	"github.com/hanwen/go-fuse/fuse/pathfs"
+)
+
+type loopBackFs struct {
+	pathfs.FileSystem
+}
+
+// A FUSE filesystem that shunts all request to an underlying file system.
+// Its main purpose is to provide test coverage.
+type hookFs struct {
+	original   string
+	mountpoint string
+	fsName     string
+	loopBackFs
+	hook Hook
+}
+
+func newHookFs(original string, mountpoint string, hook Hook) (*hookFs, error) {
+	hookfs := &hookFs{
+		original:   original,
+		mountpoint: mountpoint,
+		fsName:     "hookfs",
+		loopBackFs: loopBackFs{pathfs.NewLoopbackFileSystem(original)},
+		hook:       hook,
+	}
+	return hookfs, nil
+}
+
+// String implements hanwen/go-fuse/fuse/pathfs.FileSystem. You are not expected to call h manually.
+func (h *hookFs) String() string {
+	return fmt.Sprintf("HookFs{Original=%s, Mountpoint=%s, FsName=%s, Underlying fs=%s, hook=%s}",
+		h.original, h.mountpoint, h.fsName, h.loopBackFs.String(), h.hook)
+}
+
+// Rename implements hanwen/go-fuse/fuse/pathfs.FileSystem. You are not expected to call h manually.
+func (h *hookFs) Rename(oldName string, newName string, context *fuse.Context) fuse.Status {
+	preHooked, err := h.hook.PreRename(oldName, newName)
+	if preHooked {
+		if err != nil {
+
+			return fuse.ToStatus(err)
+		}
+	}
+
+	status := h.loopBackFs.Rename(oldName, newName, context)
+
+	postHooked, err := h.hook.PostRename(oldName, newName)
+	if postHooked {
+		if err != nil {
+			return fuse.ToStatus(err)
+		}
+	}
+	return status
+}
+
+func (h *hookFs) newServer() (*fuse.Server, error) {
+	opts := &nodefs.Options{
+		NegativeTimeout: time.Second,
+		AttrTimeout:     time.Second,
+		EntryTimeout:    time.Second,
+	}
+	pathFsOpts := &pathfs.PathNodeFsOptions{ClientInodes: true}
+	pathFs := pathfs.NewPathNodeFs(h, pathFsOpts)
+	conn := nodefs.NewFileSystemConnector(pathFs.Root(), opts)
+	originalAbs, _ := filepath.Abs(h.original)
+	mOpts := &fuse.MountOptions{
+		AllowOther: true,
+		Name:       h.fsName,
+		FsName:     originalAbs,
+	}
+	server, err := fuse.NewServer(conn.RawFS(), h.mountpoint, mOpts)
+	if err != nil {
+		return nil, err
+	}
+
+	return server, nil
+}
+
+// NewServer creates a fuse server and attaches it to the given `mountpoint` directory.
+// It returns a function to `unmount` the given `mountpoint` directory.
+func NewServer(t *testing.T, original, mountpoint string, hook Hook) (clean func()) {
+	fs, err := newHookFs(original, mountpoint, hook)
+	testutil.Ok(t, err)
+
+	server, err := fs.newServer()
+	testutil.Ok(t, err)
+
+	// Async start fuse server, and it will be stopped when calling `fuse.Unmount()` method.
+	go func() {
+		server.Serve()
+	}()
+
+	testutil.Ok(t, server.WaitMount())
+
+	return func() {
+		if err = server.Unmount(); err != nil {
+			testutil.Ok(t, err)
+		}
+
+		testutil.Ok(t, os.RemoveAll(mountpoint))
+		testutil.Ok(t, os.RemoveAll(original))
+	}
+}

testutil/fuse/fuse_windows.go

@@ -0,0 +1,21 @@
+// Copyright 2019 The qiffang Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package fuse
+
+import "testing"
+
+// NewServer is a mock method because fuse does not support Windows platform.
+func NewServer(t *testing.T, original, mountpoint string, hook Hook) (clean func()) {
+	t.Skip("Skip windows platform")
+	return func() {}
+}

testutil/fuse/hook.go

@@ -0,0 +1,57 @@
+// Copyright 2019 The qiffang Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package fuse
+
+import (
+	"syscall"
+)
+
+// Hook is the base interface for user-written hooks.
+// You have to implement XXXHooK(e.g. `TestRenameHook`, ..).
+type Hook interface {
+	// If hooked is true, the real `rename()` would not be called.
+	PreRename(oldPatgh string, newPath string) (hooked bool, err error)
+
+	// If hooked is true, it will triggered after the real `rename()`.
+	PostRename(oldPatgh string, newPath string) (hooked bool, err error)
+}
+
+// TestRenameHook is called on rename.
+type TestRenameHook struct {
+	// Add the Empty hook so the that this struct implements the hook interface.
+	EmptyHook
+}
+
+// These will take precedence over the `EmptyHook.PreRename()`.
+func (h TestRenameHook) PreRename(oldPatgh string, newPath string) (hooked bool, err error) {
+	return true, syscall.EIO
+}
+
+// These will take precedence over the `EmptyHook.PostRename()`.
+func (h TestRenameHook) PostRename(oldPatgh string, newPath string) (hooked bool, err error) {
+	return false, nil
+}
+
+// EmptyHook implements a Hook that returns false for every operation.
+type EmptyHook struct{}
+
+// PreRename is called on before real `rename()` method.
+func (h EmptyHook) PreRename(oldPatgh string, newPath string) (hooked bool, err error) {
+	return false, nil
+}
+
+// PostRename is called on after real `rename()` method.
+func (h EmptyHook) PostRename(oldPatgh string, newPath string) (hooked bool, err error) {
+	return false, nil
+}