[ISSUE #296] Support Jepsen test based on Register-Model (#297)

* feat(test): add jepsen test

1. add jepsen test

Closes #296

* feat(test): add arguments to pass in the user's name and password

1. add arguments to pass in the user's name and password

Closes #296

* build(test): add more git ignore entries

1. add more git ignore entries

Closes #296

* feat(example): add argument for read mode

1. add argument for read mode

Closes #296

.gitignore

@@ -12,4 +12,6 @@ devenv
 !NOTICE-BIN
 !LICENSE-BIN
 .DS_Store
-nohup.out
+nohup.out
+*.gz
+dledger-example.jar

example/src/main/java/io/openmessaging/storage/dledger/example/register/client/RegisterDLedgerClient.java

@@ -16,6 +16,7 @@
 
 package io.openmessaging.storage.dledger.example.register.client;
 
+import io.openmessaging.storage.dledger.ReadMode;
 import io.openmessaging.storage.dledger.client.DLedgerClient;
 import io.openmessaging.storage.dledger.example.register.protocol.RegisterReadRequest;
 import io.openmessaging.storage.dledger.example.register.protocol.RegisterReadResponse;
@@ -44,7 +45,11 @@ public RegisterWriteResponse write(int key, int value) {
     }
 
     public RegisterReadResponse read(int key) {
-        RegisterReadRequest request = new RegisterReadRequest(key);
+        return this.read(key, ReadMode.RAFT_LOG_READ);
+    }
+
+    public RegisterReadResponse read(int key, ReadMode readMode) {
+        RegisterReadRequest request = new RegisterReadRequest(key, readMode);
         return client.invokeUserDefineRequest(request, RegisterReadResponse.class, true);
     }
 

example/src/main/java/io/openmessaging/storage/dledger/example/register/command/ReadCommand.java

@@ -19,6 +19,7 @@
 import com.alibaba.fastjson.JSON;
 import com.beust.jcommander.Parameter;
 import com.beust.jcommander.Parameters;
+import io.openmessaging.storage.dledger.ReadMode;
 import io.openmessaging.storage.dledger.example.common.command.BaseCommand;
 import io.openmessaging.storage.dledger.example.register.client.RegisterDLedgerClient;
 import io.openmessaging.storage.dledger.example.register.protocol.RegisterReadResponse;
@@ -36,14 +37,18 @@ public class ReadCommand extends BaseCommand {
     @Parameter(names = {"--peers", "-p"}, description = "Peer info of this server")
     private String peers = "n0-localhost:20911";
 
-    @Parameter(names = {"--key", "-k"}, description = "the key to read")
+    @Parameter(names = {"--key", "-k"}, description = "The key to read")
     private int key = 13;
 
+    @Parameter(names = {"--read-mode"}, description = "Read mode")
+    private ReadMode readMode = ReadMode.RAFT_LOG_READ;
+
+
     @Override
     public void doCommand() {
         RegisterDLedgerClient client = new RegisterDLedgerClient(group, peers);
         client.startup();
-        RegisterReadResponse response = client.read(key);
+        RegisterReadResponse response = client.read(key, readMode);
         logger.info("Get Result:{}", JSON.toJSONString(response));
         client.shutdown();
     }

example/src/main/java/io/openmessaging/storage/dledger/example/register/protocol/RegisterReadProcessor.java

@@ -18,7 +18,6 @@
 
 import io.openmessaging.storage.dledger.DLedgerServer;
 import io.openmessaging.storage.dledger.ReadClosure;
-import io.openmessaging.storage.dledger.ReadMode;
 import io.openmessaging.storage.dledger.Status;
 import io.openmessaging.storage.dledger.protocol.userdefine.UserDefineProcessor;
 import io.openmessaging.storage.dledger.example.register.RegisterStateMachine;
@@ -49,7 +48,7 @@ public void done(Status status) {
                 }
             }
         };
-        dLedgerServer.handleRead(ReadMode.RAFT_LOG_READ, closure);
+        dLedgerServer.handleRead(registerReadRequest.getReadMode(), closure);
         return future;
     }
 

example/src/main/java/io/openmessaging/storage/dledger/example/register/protocol/RegisterReadRequest.java

@@ -16,18 +16,30 @@
 
 package io.openmessaging.storage.dledger.example.register.protocol;
 
+import io.openmessaging.storage.dledger.ReadMode;
 import io.openmessaging.storage.dledger.protocol.userdefine.UserDefineRequest;
 
 public class RegisterReadRequest extends UserDefineRequest {
 
-    private Integer key;
+    private final Integer key;
+
+    private ReadMode readMode = ReadMode.RAFT_LOG_READ;
 
     public RegisterReadRequest(int key) {
         this.key = key;
     }
 
-    public void setKey(Integer key) {
+    public RegisterReadRequest(int key, ReadMode readMode) {
         this.key = key;
+        this.readMode = readMode;
+    }
+
+    public void setReadMode(ReadMode readMode) {
+        this.readMode = readMode;
+    }
+
+    public ReadMode getReadMode() {
+        return readMode;
     }
 
     public Integer getKey() {

jepsen/build.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# build dledger
+cd ../;
+mvn clean install -DskipTests;
+mv example/target/dledger-example.jar jepsen/node-deploy/;
+rm jepsen/dledger-jepsen.tar.gz;
+chmod +x jepsen/node-deploy/startup.sh;
+chmod +x jepsen/node-deploy/stop.sh;
+chmod +x jepsen/node-deploy/stop_dropcaches.sh;
+tar zcvf jepsen/dledger-jepsen-node.tar.gz jepsen/node-deploy/dledger-example.jar jepsen/node-deploy/startup.sh jepsen/node-deploy/stop.sh jepsen/node-deploy/stop_dropcaches.sh;
+
+# build jepsen test
+cd jepsen;
+lein uberjar;
+chmod +x jepsen.sh;
+cd ../;
+tar zcvf jepsen/dledger-jepsen-control.tar.gz jepsen/jepsen.sh jepsen/nodes jepsen/target/openmessaging-dledger-jepsen-exec.jar;

jepsen/build_deploy.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+sh ./build.sh ;
+control run dledger-control control-deploy;
+control run dledger-node node-deploy;

jepsen/control.clj

@@ -0,0 +1,54 @@
+; Licensed to the Apache Software Foundation (ASF) under one or more
+; contributor license agreements.  See the NOTICE file distributed with
+; this work for additional information regarding copyright ownership.
+; The ASF licenses this file to You under the Apache License, Version 2.0
+; (the "License"); you may not use this file except in compliance with
+; the License.  You may obtain a copy of the License at
+;
+;     http://www.apache.org/licenses/LICENSE-2.0
+;
+; Unless required by applicable law or agreed to in writing, software
+; distributed under the License is distributed on an "AS IS" BASIS,
+; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+; See the License for the specific language governing permissions and
+; limitations under the License.
+
+
+(defcluster :dledger-node
+  :clients [{:host "n0" :user "root"}
+            {:host "n1" :user "root"}
+            {:host "n2" :user "root"}
+            {:host "n3" :user "root"}
+            {:host "n4" :user "root"}])
+
+(defcluster :dledger-control
+  :clients [{:host "n5" :user "root"}])
+
+(deftask :date "echo date on cluster"  []
+  (ssh "date"))
+
+(deftask :node-deploy []
+  (ssh
+   (run
+    (cd "~"
+        (run "rm -rf jepsen/")
+        (run "rm -rf dledger-jepsen-node.tar.gz"))))
+  (scp "dledger-jepsen-node.tar.gz" "~/")
+  (ssh
+   (run
+    (cd "~"
+        (run "tar zxvf dledger-jepsen-node.tar.gz")
+        (run "rm -rf dledger-jepsen-node.tar.gz")))))
+
+(deftask :control-deploy []
+  (ssh
+   (run
+    (cd "~"
+        (run "rm -rf jepsen/")
+        (run "rm -rf dledger-jepsen-control.tar.gz"))))
+  (scp "dledger-jepsen-control.tar.gz" "~/")
+  (ssh
+   (run
+    (cd "~"
+        (run "tar zxvf dledger-jepsen-control.tar.gz")
+        (run "rm -rf dledger-jepsen-control.tar.gz")))))

jepsen/jepsen.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+java -jar target/openmessaging-dledger-jepsen-exec.jar test $@

jepsen/node-deploy/startup.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+nohup java -jar ./dledger-example.jar register $@ >> register-dledger.log 2>&1 &

jepsen/node-deploy/stop.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+pid=`ps -ef |grep dledger-example |grep java |awk -F' ' '{print $2}'`
+if [ "$pid" != "" ]
+then
+    echo "kill $pid"
+    kill $pid
+fi

jepsen/node-deploy/stop_dropcaches.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+pid=`ps -ef |grep dledger-example |grep java |awk -F' ' '{print $2}'`
+if [ "$pid" != "" ]
+then
+    echo "kill $pid"
+    kill $pid
+fi
+
+# To free pagecache, dentries and inodes
+echo 3 >/proc/sys/vm/drop_caches

jepsen/nodes

@@ -0,0 +1,5 @@
+n0
+n1
+n2
+n3
+n4

jepsen/project.clj

@@ -0,0 +1,27 @@
+; Licensed to the Apache Software Foundation (ASF) under one or more
+; contributor license agreements.  See the NOTICE file distributed with
+; this work for additional information regarding copyright ownership.
+; The ASF licenses this file to You under the Apache License, Version 2.0
+; (the "License"); you may not use this file except in compliance with
+; the License.  You may obtain a copy of the License at
+;
+;     http://www.apache.org/licenses/LICENSE-2.0
+;
+; Unless required by applicable law or agreed to in writing, software
+; distributed under the License is distributed on an "AS IS" BASIS,
+; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+; See the License for the specific language governing permissions and
+; limitations under the License.
+
+
+(defproject openmessaging-dledger-jepsen "0.0.1-SNAPSHOT"
+  :descritopn "A jepsen test for DLedger"
+  :license {:name "Apache License 2.0"}
+  :main io.openmessaging.storage.dledger.jepsen.core
+  :dependencies [[org.clojure/clojure "1.10.0"]
+                 [jepsen "0.1.15-SNAPSHOT"]
+                 [io.openmessaging.storage/dledger-example "0.3.3-SNAPSHOT"]]
+  :aot [io.openmessaging.storage.dledger.jepsen.core]
+  :source-paths ["src" "src/main/clojure"]
+  :jar-name "openmessaging-dledger-jepsen.jar"
+  :uberjar-name "openmessaging-dledger-jepsen-exec.jar")