cockroachdb: no-longer rely on auto-init

When starting CockroachDB nodes, always include join flags. In order to
actually initialize the cluster, rely on an explicit `cockroach init`.

Previously any node started without explicit join flags was tasked with
bootstrapping the cluster. This was deprecated behavior, and was removed
in cockroachdb/cockroach#51245.

Fixes cockroachdb/cockroach#51050
Fixes cockroachdb/cockroach#51052
Fixes cockroachdb/cockroach#51725
Fixes cockroachdb/cockroach#51727
Fixes cockroachdb/cockroach#51726
Fixes cockroachdb/cockroach#51729
Fixes cockroachdb/cockroach#51730
Fixes cockroachdb/cockroach#51728
Fixes cockroachdb/cockroach#51731
Fixes cockroachdb/cockroach#51733
Fixes cockroachdb/cockroach#51732
Fixes cockroachdb/cockroach#51734
Fixes cockroachdb/cockroach#51735
Fixes cockroachdb/cockroach#51736
Fixes cockroachdb/cockroach#51740
Fixes cockroachdb/cockroach#51737
Fixes cockroachdb/cockroach#51741
Fixes cockroachdb/cockroach#51742
Fixes cockroachdb/cockroach#51746
Fixes cockroachdb/cockroach#51745
Fixes cockroachdb/cockroach#51744
Fixes cockroachdb/cockroach#51743
Fixes cockroachdb/cockroach#51747
Fixes cockroachdb/cockroach#51755
Fixes cockroachdb/cockroach#51754
Fixes cockroachdb/cockroach#51753
Fixes cockroachdb/cockroach#51752
Fixes cockroachdb/cockroach#51750
Fixes cockroachdb/cockroach#51751
Fixes cockroachdb/cockroach#51749
Fixes cockroachdb/cockroach#49360
Fixes cockroachdb/cockroach#51756
Fixes cockroachdb/cockroach#51748
Fixes cockroachdb/cockroach#51758

cockroachdb/src/jepsen/cockroach.clj

@@ -70,6 +70,10 @@
                   (auto/start! test node)
                   (Thread/sleep 5000)) ; Give it time to join
 
+                (when (= node (jepsen/primary test))
+                  (auto/init! node)
+                  (Thread/sleep 5000))
+
                 (jepsen/synchronize test)
                 (when (= node (jepsen/primary test))
                   (auto/set-replication-zone! "default"

cockroachdb/src/jepsen/cockroach/auto.clj

@@ -57,6 +57,13 @@
            ]
           (if insecure [:--insecure] [])))
 
+;; Extra command-line arguments to give to `cockroach init`
+(def cockroach-init-arguments
+  (concat [:init
+           ;; ... other arguments here ...
+           ]
+          (if insecure [:--insecure] [])))
+
 (defn control-addr
   "Address of the Jepsen control node, as seen by the local node. Used to
   filter packet captures."
@@ -162,16 +169,21 @@
    extra-args
    [:--logtostderr :>> errlog (c/lit "2>&1")]))
 
-(defn runcmd
-  "The command to run cockroach for a given test"
-  [test node joining?]
-  (let [join (if joining?
-               [(->> (:nodes test)
+(defn cockroach-init-cmdline
+  "Construct the command line to initialize a CockroachDB cluster."
+  []
+  (concat
+   [(c/expand-path cockroach)]
+   cockroach-init-arguments))
+
+(defn startcmd
+  "The command to start cockroach for a given test"
+  [test node]
+  (let [join [(->> (:nodes test)
                      (remove #{node})
                      (map name)
                      (str/join ",")
-                     (str "--join="))]
-               [])]
+                     (str "--join="))]]
     (wrap-env [(str "COCKROACH_LINEARIZABLE="
                     (if (:linearizable test) "true" "false"))
                (str "COCKROACH_MAX_OFFSET=" "250ms")]
@@ -186,11 +198,24 @@
                          (catch RuntimeException e "")))
             (info node "Cockroach already running.")
             (do (info node "Starting CockroachDB...")
-                (c/trace (c/exec (runcmd test node
-                                         (not= node (jepsen/primary test)))))
+                (c/trace (c/exec (startcmd test node)))
                 (info node "Cockroach started"))))
   :started)
 
+(defn initcmd
+  "The command to initialize cockroach for a given test"
+  []
+  (cockroach-init-cmdline))
+
+(defn init!
+  "Initialize cockroachdb cluster"
+  [node]
+  (c/sudo cockroach-user
+          (do (info node "Initializing CockroachDB...")
+              (c/trace (c/exec (initcmd)))
+              (info node "Cockroach initialized")))
+  :initialized)
+
 (defn kill!
   "Kills cockroach on node."
   [test node]