*: support custom content check online in v2store

It's to use v2 API to list all keys in v2store. If there is any active
key in v2store, `etcdctl check v2store` returns error. However, the last
snapshot might still contain custom content. The end-user still needs to
check that content offline by `etcdutl check v2store`.

Part of #18993

Signed-off-by: Wei Fu <[email protected]>

etcdctl/ctlv3/command/check.go

@@ -113,6 +113,7 @@ func NewCheckCommand() *cobra.Command {
 
 	cc.AddCommand(NewCheckPerfCommand())
 	cc.AddCommand(NewCheckDatascaleCommand())
+	cc.AddCommand(NewCheckV2StoreCommand())
 
 	return cc
 }
@@ -437,3 +438,38 @@ func newCheckDatascaleCommand(cmd *cobra.Command, args []string) {
 		fmt.Println(fmt.Sprintf("PASS: Approximate system memory used : %v MB.", strconv.FormatFloat(mbUsed, 'f', 2, 64)))
 	}
 }
+
+// NewCheckV2StoreCommand returns the cobra command for "check v2store".
+func NewCheckV2StoreCommand() *cobra.Command {
+	return &cobra.Command{
+		Use:   "v2store",
+		Short: "Check custom content in v2store memory",
+		Run:   checkV2StoreMemoryRunFunc,
+	}
+}
+
+func checkV2StoreMemoryRunFunc(cmd *cobra.Command, _ []string) {
+	err := checkV2StoreMemory(cmd)
+	if err != nil {
+		cobrautl.ExitWithError(cobrautl.ExitError, err)
+	}
+}
+
+func checkV2StoreMemory(cmd *cobra.Command) error {
+	scfg := secureCfgFromCmd(cmd)
+
+	cli := clientConfigFromCmd(cmd).mustClient()
+	ep := cli.Endpoints()[0]
+
+	res, err := listAllKeysFromV2Store(ep, scfg)
+	if err != nil {
+		return err
+	}
+
+	if len(res.Node.Nodes) > 0 {
+		return fmt.Errorf("detected custom content in v2store memory")
+	}
+
+	fmt.Println("No active custom content in v2store memory. Still need to run `etcdutl check v2store` offline to make sure that there is no custom content in WAL")
+	return nil
+}

etcdctl/ctlv3/command/util.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"crypto/tls"
 	"encoding/hex"
+	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"net/http"
@@ -27,6 +28,7 @@ import (
 	"time"
 
 	pb "go.etcd.io/etcd/api/v3/mvccpb"
+	clientv2 "go.etcd.io/etcd/client/v2"
 	v3 "go.etcd.io/etcd/client/v3"
 	"go.etcd.io/etcd/pkg/v3/cobrautl"
 
@@ -166,3 +168,45 @@ func defrag(c *v3.Client, ep string) {
 	}
 	fmt.Printf("Defragmented %q\n", ep)
 }
+
+// listAllKeysFromV2Store lists all keys in v2store memory.
+func listAllKeysFromV2Store(host string, scfg *secureCfg) (*clientv2.Response, error) {
+	if !strings.HasPrefix(host, "http://") && !strings.HasPrefix(host, "https://") {
+		host = "http://" + host
+	}
+
+	if strings.HasPrefix(host, "https://") {
+		cert, err := tls.LoadX509KeyPair(scfg.cert, scfg.key)
+		if err != nil {
+			return nil, fmt.Errorf("client certificate error: %w", err)
+		}
+
+		http.DefaultTransport.(*http.Transport).TLSClientConfig = &tls.Config{
+			Certificates:       []tls.Certificate{cert},
+			InsecureSkipVerify: scfg.insecureSkipVerify,
+		}
+	}
+
+	kurl := host + "/v2/keys/?recursive=true"
+
+	resp, err := http.Get(kurl)
+	if err != nil {
+		return nil, fmt.Errorf("fetch %s error: %w", kurl, err)
+	}
+	defer resp.Body.Close()
+
+	bytes, rerr := ioutil.ReadAll(resp.Body)
+	if rerr != nil {
+		return nil, fmt.Errorf("read %s error: %w", kurl, rerr)
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("fetch %s with unexpected code %d: %s", kurl, resp.StatusCode, string(bytes))
+	}
+
+	var res clientv2.Response
+	if err := json.Unmarshal(bytes, &res); err != nil {
+		return nil, fmt.Errorf("failed to unmarshal %s: %w", string(bytes), err)
+	}
+	return &res, nil
+}

tests/e2e/v2store_deprecation_test.go

@@ -41,6 +41,14 @@ func createV2store(t testing.TB, dataDirPath string) {
 			t.Fatalf("failed put with curl (%v)", err)
 		}
 	}
+
+	t.Log("Verify keys in v2store memory")
+	epURL := epc.Procs[0].EndpointsV3()[0]
+	proc, err := e2e.SpawnCmd([]string{e2e.BinDir + "/etcdctl", "--endpoints=" + epURL, "check", "v2store"}, nil)
+	assert.NoError(t, err)
+
+	_, err = proc.Expect("detected custom content in v2store memory")
+	assert.NoError(t, err)
 }
 
 func assertVerifyCanStartV2deprecationNotYet(t testing.TB, dataDirPath string) {